An Exploration of Color Recognition through Clustering

import ast
import binascii
from itertools import repeat
import json
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from PIL import Image
import requests
import scipy.cluster
import scipy.stats as ss
import sqlite3
from sqlalchemy import create_engine

Let’s write a small function to query the Art Institute of Chicago’s Artworks API, and test with a random integer.

In [2]:

def get_art_attributes(_id):
url = (
f"https://api.artic.edu/api/v1/artworks/{_id}?"
f"fields=id,image_id,date_end,place_of_origin,artwork_type_title"
)
r = requests.get(url)
data = json.loads(r.text)['data']
return data
get_art_attributes(897)

{'id': 897,
'date_end': 1852,
'place_of_origin': 'France',
'artwork_type_title': 'Painting',
'image_id': '5ae91cbf-66c5-cf9b-f355-629e458cb063'}

In [3]:

def get_art_image(image_id):
im = None
url = (
f"https://www.artic.edu/iiif/2/"
f"{image_id}"
f"/full/843,/0/default.jpg"
)
try:
im = Image.open(requests.get(url, stream=True).raw)
except:
pass
return im.resize((300, 300))
get_art_image(get_art_attributes(893)['image_id'])

Out[3]: Excellent! Now we have three very important data points: time, place, and the painting itself. Let’s run this through our color clustering algorithm that we developed a couple weeks ago.

def get_color_stats(im=None, ar=None):
if im and not ar:
ar = np.asarray(im)
shape = ar.shape
ar = ar.reshape(np.prod(shape[:2]), shape[2]).astype(float)
codes, dist = scipy.cluster.vq.kmeans(ar, 4)
vectors, distance = scipy.cluster.vq.vq(ar, codes)
counts, bins = np.histogram(vectors, len(codes))
colors = dict(zip(ss.rankdata(-counts), codes.tolist()))
colors = {int(k): {'rgb': v} for k, v in colors.items()}

for i, v in enumerate(colors):
colors[v]['count'] = counts[i]
for v in colors.values():
v['rgb'] = [round(n) for n in v['rgb']]
v['hex'] = f"#{binascii.hexlify(bytearray(int(c) for c in v['rgb'])).decode('ascii')}"
v['r'] = v['rgb'][0]
v['g'] = v['rgb'][1]
v['b'] = v['rgb'][2]
df = pd.DataFrame.from_dict
data=colors, orient='index').reset_index().rename(
columns={'index': 'rank'}).sort_values(by='rank')
return df, ar
df, ar = get_color_stats(get_art_image(get_art_attributes(893)['image_id']))
df

def plot_rgb(ar,s=0.1):
X = np.hsplit(ar, np.array([1, 2]))[0].flatten().tolist()
Y = np.hsplit(ar, np.array([1, 2]))[1].flatten().tolist()
Z = np.hsplit(ar, np.array([1, 2]))[2].flatten().tolist()
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(111, projection = '3d')
ax.scatter(X, Y, Z, s=s, c=ar / 255.0)
plt.show()
plot_rgb(ar)

def create_mem_sqlite():
engine = create_engine('sqlite://', echo=False)
return engine
def load_to_sqlite(df, name, engine):
df.to_sql(name, con=engine)
def workflow(_id):
data = {}
try:
data = get_art_attributes(_id)
if 'image_id' in data:
im = get_art_image(data['image_id'])
if im:
df, ar = get_color_stats(im)
data['colors'] = str(df['hex'].to_list())
data['counts'] = str(df['count'].to_list())
except KeyError:
pass
return data

engine = create_mem_sqlite()
paintings = {}
for _id in range(890, 900):
data = workflow(_id)
if data:
paintings[_id] = data
paintings = {k: v for k, v in paintings.items() if v}

df = pd.DataFrame.from_dict(paintings, orient='index')

load_to_sqlite(df, 'paintings', engine)

pd.read_sql_query("SELECT * FROM paintings", engine).head(5)

ar = np.empty(shape=[1, 3])
for image_id in [i for i in pd.read_sql_query("SELECT image_id FROM paintings", engine)['image_id'].to_list() if i]:
im = get_art_image(image_id)
im = im.resize((300, 300))
_ar = np.asarray(im)
shape = _ar.shape
_ar = _ar.reshape(np.prod(shape[:2]), shape[2]).astype(float)
ar = np.append(ar, _ar , axis=0)
plot_rgb(ar)

Take a close look at this plot

What strikes you about it? It’s crammed to the gills with data points.

Remember that our ultimate goal is to analyze dominant colors across multiple artworks. So it seems we have two options:

1. Add all pixels from all paintings into one 3D space, then find dominant color clusters.
2. For each painting , extract dominant pixel cluster center and count. Add these to 3D space, weigh by count, and find dominant color clusters within that space.

Let’s compare the results of these two methods and talk through the pros and cons of each.

option1_df, option1_ar = get_color_stats(ar=ar)

option2_raw = pd.read_sql_query("SELECT * FROM paintings", engine)
option2_raw['colors'] = option2_raw['colors'].apply(lambda x: ast.literal_eval(x))
option2_raw['colors'] = option2_raw['colors'].apply(lambda x: [list(int(h.replace('#', '')[i:i+2], 16) for i in (0, 2, 4)) for h in x])
option2_raw['counts'] = option2_raw['counts'].apply(lambda x: ast.literal_eval(x))

my_array = []
for c, n in zip(option2_raw['colors'].to_list(), option2_raw['counts'].to_list()):
for ci, ni in zip(c, n):
my_array.extend(repeat(ci, ni))
my_array = np.array([[float(number) for number in group] for group in my_array])

option2_df, option2_ar = get_color_stats(ar=my_array)

print(option1_df)
print('*'*60)
print(option2_df)

figure, axis = plt.subplots(1, 2, figsize=(10,5))

axis[0].bar(
x=option1_df['hex'],
height=option1_df['count'],
color=option1_df['hex'].tolist())
axis[0].set_title("Option 1: All Pixels in one Space")
axis[1].bar(
x=option1_df['hex'],
height=option2_df['count'],
color=option1_df['hex'].tolist())
axis[1].set_title("Option 2: Weighted Dominant Colors from Each Painting")
plt.show()