cs231n：assignment1——Q3: Implement a Softmax classifier （MemoryError: ）

喜欢ヅ旅行 2022-04-17 04:45 155阅读 0赞

在这个作业里，当运行第一步的获取数据的时候，出现了MemoryError: 的错误，主要原因如下：

加载数据代码：

import random  
import numpy as np  
from cs231n.data\_utils import load\_CIFAR10  
import matplotlib.pyplot as plt

from \_\_future\_\_ import print\_function

%matplotlib inline  
plt.rcParams\['figure.figsize'\] = (10.0, 8.0) \# set default size of plots  
plt.rcParams\['image.interpolation'\] = 'nearest'  
plt.rcParams\['image.cmap'\] = 'gray'

\# for auto-reloading extenrnal modules  
\# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython  
%load\_ext autoreload  
%autoreload 2

def get\_CIFAR10\_data(num\_training=49000, num\_validation=1000, num\_test=1000, num\_dev=500):  
    """  
    Load the CIFAR-10 dataset from disk and perform preprocessing to prepare  
    it for the linear classifier. These are the same steps as we used for the  
    SVM, but condensed to a single function.    
    """  
    \# Load the raw CIFAR-10 data  
    cifar10\_dir = 'cs231n/datasets/cifar-10-batches-py'  
      
    X\_train, y\_train, X\_test, y\_test = load\_CIFAR10(cifar10\_dir)  
      
    \# subsample the data  
    mask = list(range(num\_training, num\_training + num\_validation))  
    X\_val = X\_train\[mask\]  
    y\_val = y\_train\[mask\]  
    mask = list(range(num\_training))  
    X\_train = X\_train\[mask\]  
    y\_train = y\_train\[mask\]  
    mask = list(range(num\_test))  
    X\_test = X\_test\[mask\]  
    y\_test = y\_test\[mask\]  
    mask = np.random.choice(num\_training, num\_dev, replace=False)  
    X\_dev = X\_train\[mask\]  
    y\_dev = y\_train\[mask\]  
      
    \# Preprocessing: reshape the image data into rows  
    X\_train = np.reshape(X\_train, (X\_train.shape\[0\], -1))  
    X\_val = np.reshape(X\_val, (X\_val.shape\[0\], -1))  
    X\_test = np.reshape(X\_test, (X\_test.shape\[0\], -1))  
    X\_dev = np.reshape(X\_dev, (X\_dev.shape\[0\], -1))  
      
    \# Normalize the data: subtract the mean image  
    mean\_image = np.mean(X\_train, axis = 0)  
    X\_train -= mean\_image  
    X\_val -= mean\_image  
    X\_test -= mean\_image  
    X\_dev -= mean\_image  
      
    \# add bias dimension and transform into columns  
    X\_train = np.hstack(\[X\_train, np.ones((X\_train.shape\[0\], 1))\])  
    X\_val = np.hstack(\[X\_val, np.ones((X\_val.shape\[0\], 1))\])  
    X\_test = np.hstack(\[X\_test, np.ones((X\_test.shape\[0\], 1))\])  
    X\_dev = np.hstack(\[X\_dev, np.ones((X\_dev.shape\[0\], 1))\])  
      
    return X\_train, y\_train, X\_val, y\_val, X\_test, y\_test, X\_dev, y\_dev

\# Cleaning up variables to prevent loading data multiple times (which may cause memory issue)  
try:  
   del X\_train, y\_train  
   del X\_test, y\_test  
   print('Clear previously loaded data.')  
except:  
   pass

\# Invoke the above function to get our data.  
X\_train, y\_train, X\_val, y\_val, X\_test, y\_test, X\_dev, y\_dev = get\_CIFAR10\_data()  
print('Train data shape: ', X\_train.shape)  
print('Train labels shape: ', y\_train.shape)  
print('Validation data shape: ', X\_val.shape)  
print('Validation labels shape: ', y\_val.shape)  
print('Test data shape: ', X\_test.shape)  
print('Test labels shape: ', y\_test.shape)  
print('dev data shape: ', X\_dev.shape)  
print('dev labels shape: ', y\_dev.shape)

运行后发现了  MemoryError:

问题出在：

data\_utils.py 中

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzI5NzUwNDYx_size_16_color_FFFFFF_t_70][]

如上所示

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzI5NzUwNDYx_size_16_color_FFFFFF_t_70]: /images/20220417/ad2d13ae13d847ba8ce791cc753e163c.png