I have created a function, imgs_to_df() (which relies on img_to_vec()) that takes a list of URLs that point to a JPG (e.g. https://live.staticflickr.com/65535/48123413937_54bb53e98b_o.jpg), resizes it, and converts the URLs to a dataframe of RGB values, where each row is a different image, and each column is the R, G, or B value of the pixel of the (resized) image.
However, the function is very slow, especially once it gets into lists of hundreds or thousands of links, so I need a way to parallelize or otherwise make the process much, much faster. I'd also like to ensure there is a way to easily to match the URLs back with the RGB vectors after I'm done.
I am very new to parallel processing and everything I have read is just confusing me even more.
from PIL import Image
from io import BytesIO
import urllib.request
import requests
import numpy as np
import pandas as pd
def img_to_vec(jpg_url, resize=True, new_width=300, new_height=300):
""" Takes a URL of an image, resizes it (optional), and converts it to a
vector representing RGB values.
Parameters
----------
jpg_url: String. A URL that points to a JPG image.
resize: Boolean. Default True. Whether image should be resized before calculating RGB.
new_width: Int. Default 300. New width to convert image to before calculating RGB.
new_height: Int. Default 300. New height to conver image to before calculating RGB.
Returns
-------
rgb_vec: Vector of size 3*new_width*new_height for the RGB values in each pixel of the image.
"""
response = requests.get(jpg_url) # Create remote image connection
img = Image.open(BytesIO(response.content)) # Save image connection (NOT actual image)
if resize:
img = img.resize((new_width, new_height))
rgb_img = np.array(img) # Create matrix of RGB values
rgb_vec = rgb_img.ravel() # Flatten 3D matrix of RGB values to a vector
return rgb_vec
# Consider parallel processing here
def imgs_to_df(jpg_urls, common_width=300, common_height=300):
""" Takes a list of jpg_urls and converts it to a dataframe of RGB values.
Parameters
----------
jpg_urls: A list of jpg_urls to be resized and converted to a dataframe of RGB values.
common_width: Int. Default 300. New width to convert all images to before calculating RGB.
common_height: Int. Default 300. New height to convert all images to before calculating RGB.
Returns
-------
rgb_df: Pandas dataframe of dimensions len(jpg_urls) rows and common_width*common_height*3
columns. Each row is a unique jpeg image, and each column is an R/G/B value of
a particular pixel of the resized image
"""
assert common_width>0 and common_height>0, 'Error: invalid new_width or new_height dimensions'
for url_idx in range(len(jpg_urls)):
if url_idx % 100 == 0:
print('Converting url number {urlnum} of {urltotal} to RGB '.format(urlnum=url_idx, urltotal=len(jpg_urls)))
try:
img_i = img_to_vec(jpg_urls[url_idx])
if url_idx == 0:
vecs = img_i
else:
try:
vecs = np.vstack((vecs, img_i))
except:
vecs = np.vstack((vecs, np.array([-1]*common_width*common_height*3)))
print('Warning: Error in converting {error_url} to RGB'.format(error_url=jpg_urls[url_idx]))
except:
vvecs = np.vstack((vecs, np.array([-1]*common_width*common_height*3)))
print('Warning: Error in converting {error_url} to RGB'.format(error_url=jpg_urls[url_idx]))
rgb_df = pd.DataFrame(vecs)
return rgb_df