Plotting a large database in Basemap: Memory error

Viewed 38

Hi I extract the data from an interpolation (The data are in the Basemap Grid) with the command and save them as CSV:

def inter_todf(interpolation, grid):
    grid['x'], grid['y'] = basemap(grid['x'],grid['y'],inverse=True) # Wandelt Grid in Long und Lat wieder um 
    dfl = pd.DataFrame({
        'Latitude': grid['y'].reshape(-1),
        'Longitude': grid['x'].reshape(-1),
        'Value': interpolation.reshape(-1)
        });
    return(dfl)

dfl= inter_todf(interpolation, grid)
dfl.to_csv(plot_folder+'Dezember/'+file.replace(".csv", "grid.csv"))

Afterwards I want to plot them again in another file in Basemap (so it is still planned that I modify the data but that is something else). I do the plotting with the code:

from traceback import print_tb
import numpy as np
from pykrige.ok import OrdinaryKriging
from pykrige.kriging_tools import write_asc_grid
import pykrige.kriging_tools as kt
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from matplotlib.patches import Path, PathPatch
import pandas as pd
def load_data():
    df = pd.read_csv(r"GridFile.csv")
    return(df)

def get_data(df):
    return {
        "lons": df['Longitude'].values.reshape(52920,),
        "lats": df['Latitude'].values.reshape(52920,),
        "values": df['Value'].values.reshape(52920,)
    }

def extend_data(data):
    return {
        "lons": np.concatenate([np.array([lon-360 for lon in data["lons"]]), data["lons"], np.array([lon+360 for lon in data["lons"]])]),
        "lats":  np.concatenate([data["lats"], data["lats"], data["lats"]]),
        "values":  np.concatenate([data["values"], data["values"], data["values"]]),
    }

def generate_grid(data, basemap, delta=1):
    grid = {
        'lon': np.arange(-180, 180, delta),
        'lat': np.arange(-89.9, 89.9, delta) 
    }
    grid["x"], grid["y"] = np.meshgrid(grid["lon"], grid["lat"])
    grid["x"], grid["y"] = basemap(grid["x"], grid["y"])
    return grid

def interpolate(data, grid):
    OK =OrdinaryKriging(
        data["lons"],
        data["lats"],
        data["values"],
        variogram_model='exponential',
    )
    return OK.execute("grid", grid["lon"], grid["lat"])

def prepare_map_plot():
    figure, axes = plt.subplots(figsize=(10,10))
    basemap = Basemap(projection='robin', lon_0=0, lat_0=0, resolution='l',area_thresh=1000000,ax=axes) 
    return figure, axes, basemap

def plot_mesh_data(interpolation, grid, basemap):
    colormesh = basemap.contourf(grid["x"], grid["y"],  interpolation,32, cmap='RdBu_r', ) #plot the data on the map. plt.cm.RdYlBu_r
    color_bar = basemap.colorbar(colormesh,location='bottom',pad="10%") 


df = load_data()
base_data = get_data(df)
# print(df['Latitude'].shape)
figure, axes, basemap = prepare_map_plot()
grid = generate_grid(base_data, basemap, 40)
extended_data = extend_data(base_data)
interpolation, interpolation_error = interpolate(extended_data, grid)
plot_mesh_data(interpolation, grid,basemap)
plt.show()

Unfortunately I get an error message:

numpy.core._exceptions.MemoryError: Unable to allocate 93.9 GiB for an array with shape (12602289420,) and data type float64

How do I have to change my data so that it no longer uses so much memory?

0 Answers
Related