Weighted network between cluster centroids - Python

Before solving the problem, I guess the KMeans clustering should be performed for the first two columns of the df and the other two columns separately. If you apply KMeans clustering directly to the entire df, the connections between two clusters (A,B) and (C,D) will be trivial.

If it is possible to use networkx in your project, here is how you can achieve what you are looking for. First, prepare the data

import numpy as np
import pandas as pd
from collections import Counter
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans


def kmeans(arr,num_clusters):
    
    model = KMeans(n_clusters = num_clusters)
    model.fit(arr)
    model.cluster_centers_ 
    cluster_centers = model.cluster_centers_
    all_labels = model.labels_
    mem_count = Counter(all_labels)
    
    return cluster_centers,all_labels,mem_count
    
df = pd.DataFrame(np.random.randint(-80,80,size=(200, 4)), columns=list('ABCD'))
# tail 
A,B = df['A'],df['B']
# end 
C,D = df['C'],df['D']

nclusters_1,nclusters_2 = 20,20
points= df[['A','B','C','D']].values
cluster_one = kmeans(points[:,:2],nclusters_1)
cluster_two = kmeans(points[:,2:],nclusters_2)

# find connections between clusters 
all_combs = [[n1,n2] for n1 in range(nclusters_1) for n2 in range(nclusters_2)]
num_connections = {}
for item in all_combs:
    l1,l2 = cluster_one[1],cluster_two[1]
    mask1 = np.where(l1==item[0])[0]
    mask2 = np.where(l2==item[1])[0]
    num_common = len(list(set(mask1).intersection(mask2)))
    num_connections[(item[0],item[1]+nclusters_1)] = num_common

As you can see in the above code, I performed KMeans clustering to (A,B) and (C,D) separately, and you can use different number of clusters for the two sets of points by using different nclusters_1 and nclusters_2. After the data is prepared, we can now visualize it using networkx

# plot the graph 
import networkx as nx

G = nx.Graph()
node_sizes = {}
node_colors = {}
for k,v in num_connections.items():
    # the number of points in the two clusters 
    s1,s2 = cluster_one[2][k[0]],cluster_two[2][k[1]-nclusters_1]
    G.add_node(k[0],pos=cluster_one[0][k[0]])
    G.add_node(k[1],pos=cluster_two[0][k[1]-nclusters_1])
    G.add_edge(k[0],k[1],color='k',weight=v/3)
    node_sizes[k[0]] = s1;node_sizes[k[1]] = s2
    node_colors[k[0]] = 'navy';node_colors[k[1]] = 'darkviolet'
    
edges = G.edges()
d = dict(G.degree)
pos=nx.get_node_attributes(G,'pos')
weights = [G[u][v]['weight'] for u,v in edges]
nx.draw(G,pos,edges=edges,
        node_color=[node_colors[v] for v in d.keys()],
        nodelist=d.keys(),
        width=weights,
        node_size=[node_sizes[v]*20 for v in d.keys()])

The output figure looks like

In this figure, the actual positions of the points are used for plotting, (A,B) are colored navy and (C,D) are colored violet. If you want to scale up the node size, just use a different number than 20 for this line

node_size=[node_sizes[v]*20 for v in d.keys()]

In order to adjust the width of the edges, use can use a different number other than 3 in this line

G.add_edge(k[0],k[1],color='k',weight=v/3)

UPDATE

In the script above, the keys of num_connections represent two graph nodes, and the corresponding values represent the number of connections. In order to extract adjacency matrix from num_connections, you can try

adj_mat = np.zeros((nclusters_1,nclusters_2))
for k,v in num_connections.items():
    entry = (k[0],k[1]-nclusters_1)
    adj_mat[entry] = v

UPDATE 2

To resolve the issue in OP's updated post,

1. add this pos=nx.get_node_attributes(G,'pos') to fix the node positions
2. change G.add_node(k[0],pos=points[:,:2][k[0]]) and G.add_node(k[1],pos=points[:,2:][k[1]]) to G.add_node(k[0],pos=cluster_one[0][k[0]]) and G.add_node(k[1],pos=cluster_two[0][k[1]-nclusters_1])

with these two modifications,

df = pd.DataFrame({
    'A' : [5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 5, 4],                        
    'B' : [5, 5, 5, 5, 5, 2, 2, 2, 2, 2, 5, 2],              
    'C' : [7, 7, 7, 7, 7, 3, 3, 3, 3, 3, 3, 7],   
    'D' : [7, 7, 7, 7, 7, 4, 4, 4, 4, 4, 4, 7],                                    
    })

fig,ax = plt.subplots()
ax.set_xlim(0, 10)
ax.set_ylim(0, 10)

A = df['A']
B = df['B']

C = df['C']
D = df['D']

def kmeans(arr,num_clusters):

    model = KMeans(n_clusters = num_clusters)
    model.fit(arr)
    model.cluster_centers_ 
    cluster_centers = model.cluster_centers_
    all_labels = model.labels_
    mem_count = Counter(all_labels)

    return cluster_centers,all_labels,mem_count

nclusters_1,nclusters_2 = 2,2
points= df[['A','B','C','D']].values
cluster_one = kmeans(points[:,:2],nclusters_1)
cluster_two = kmeans(points[:,2:],nclusters_2)

# find connections between clusters 
all_combs = [[n1,n2] for n1 in range(nclusters_1) for n2 in range(nclusters_2)]
num_connections = {}
for item in all_combs:
    l1,l2 = cluster_one[1],cluster_two[1]
    mask1 = np.where(l1==item[0])[0]
    mask2 = np.where(l2==item[1])[0]
    num_common = len(list(set(mask1).intersection(mask2)))
    num_connections[(item[0],item[1]+nclusters_1)] = num_common

G = nx.Graph()
node_sizes = {}
node_colors = {}
for k,v in num_connections.items():
    # the number of points in the two clusters 
    s1,s2 = cluster_one[2][k[0]],cluster_two[2][k[1]-nclusters_1]
    G.add_node(k[0],pos=cluster_one[0][k[0]])
    G.add_node(k[1],pos=cluster_two[0][k[1]-nclusters_1])
    G.add_edge(k[0],k[1],color='k',weight=v/3)
    node_sizes[k[0]] = s1;node_sizes[k[1]] = s2
    node_colors[k[0]] = 'k';node_colors[k[1]] = 'k'

edges = G.edges()
d = dict(G.degree)
pos=nx.get_node_attributes(G,'pos')
weights = [G[u][v]['weight'] for u,v in edges]
nx.draw(G,pos,edges=edges,
        node_color=[node_colors[v] for v in d.keys()],
        nodelist=d.keys(),
        width=weights,
        node_size=[node_sizes[v]*20 for v in d.keys()])