{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datamule as dm\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import networkx as nx\n",
    "import plotly.graph_objects as go"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading indices at 2024-09-14 15:38:11.138123\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "data\\metadata.json: 100%|██████████| 59.0/59.0 [00:00<?, ?iB/s]\n",
      "data\\company_tickers.csv: 398kiB [00:00, 2.73MiB/s]\n",
      "data\\submissions_index.csv: 961MiB [00:20, 49.8MiB/s] \n"
     ]
    }
   ],
   "source": [
    "indexer = dm.Indexer()\n",
    "\n",
    "indexer.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Takes 14 seconds to run\n",
    "df = pd.read_csv('data/submissions_index.csv')\n",
    "df_8k = df[df['form'] == '8-K']\n",
    "del df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accession_number</th>\n",
       "      <th>filing_date</th>\n",
       "      <th>primary_doc_url</th>\n",
       "      <th>form</th>\n",
       "      <th>cik</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>0001193125-24-199396</td>\n",
       "      <td>2024-08-13</td>\n",
       "      <td>d864520d8k.htm</td>\n",
       "      <td>8-K</td>\n",
       "      <td>1827087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>0001193125-24-183318</td>\n",
       "      <td>2024-07-24</td>\n",
       "      <td>d875446d8k.htm</td>\n",
       "      <td>8-K</td>\n",
       "      <td>1827087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>0001193125-24-180727</td>\n",
       "      <td>2024-07-18</td>\n",
       "      <td>d870519d8k.htm</td>\n",
       "      <td>8-K</td>\n",
       "      <td>1827087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>0001193125-24-170520</td>\n",
       "      <td>2024-06-27</td>\n",
       "      <td>d792733d8k.htm</td>\n",
       "      <td>8-K</td>\n",
       "      <td>1827087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>0001193125-24-156044</td>\n",
       "      <td>2024-06-06</td>\n",
       "      <td>d675020d8k.htm</td>\n",
       "      <td>8-K</td>\n",
       "      <td>1827087</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         accession_number filing_date primary_doc_url form      cik\n",
       "139  0001193125-24-199396  2024-08-13  d864520d8k.htm  8-K  1827087\n",
       "141  0001193125-24-183318  2024-07-24  d875446d8k.htm  8-K  1827087\n",
       "142  0001193125-24-180727  2024-07-18  d870519d8k.htm  8-K  1827087\n",
       "143  0001193125-24-170520  2024-06-27  d792733d8k.htm  8-K  1827087\n",
       "151  0001193125-24-156044  2024-06-06  d675020d8k.htm  8-K  1827087"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect data\n",
    "df_8k.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15630, 5)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Subset by accession_numbers which have more than one cik\n",
    "joint_filers_df = df_8k[df_8k.groupby('accession_number')['cik'].transform('nunique') > 1]\n",
    "joint_filers_df.head()\n",
    "joint_filers_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       accession_number filing_date      primary_doc_url form     cik ticker  \\\n",
      "0  0001104659-24-082494  2024-07-25  tm2420031d1_8-k.htm  8-K  811156    CMS   \n",
      "1  0001104659-24-058008  2024-05-07   tm2413710d1_8k.htm  8-K  811156    CMS   \n",
      "2  0001104659-24-051509  2024-04-25   tm2412653d1_8k.htm  8-K  811156    CMS   \n",
      "3  0001104659-24-009113  2024-02-01    tm244795d1_8k.htm  8-K  811156    CMS   \n",
      "4  0001104659-23-122055  2023-11-29   tm2331708d1_8k.htm  8-K  811156    CMS   \n",
      "\n",
      "             title  \n",
      "0  CMS ENERGY CORP  \n",
      "1  CMS ENERGY CORP  \n",
      "2  CMS ENERGY CORP  \n",
      "3  CMS ENERGY CORP  \n",
      "4  CMS ENERGY CORP  \n",
      "(15630, 7)\n",
      "49\n"
     ]
    }
   ],
   "source": [
    "# Load company tickers\n",
    "tickers = pd.read_csv('data/company_tickers.csv')\n",
    "unique_tickers = tickers.drop_duplicates(subset='cik', keep='first')\n",
    "\n",
    "# Merge with joint_filers by CIK\n",
    "joint_filers_tickers = joint_filers_df.merge(unique_tickers, on='cik', how='left')\n",
    "\n",
    "# Display the first few rows of the merged dataframe\n",
    "print(joint_filers_tickers.head())\n",
    "\n",
    "# Display the shape of the merged dataframe\n",
    "print(joint_filers_tickers.shape)\n",
    "\n",
    "# number of unique ciks\n",
    "print(joint_filers_tickers['cik'].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jgfri\\AppData\\Local\\Temp\\ipykernel_16764\\815276929.py:32: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.\n",
      "  plt.tight_layout()\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 627768 characters]",
      "text/plain": [
       "<Figure size 2000x1600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of nodes (CIKs): 49\n",
      "Number of edges (connections): 59\n",
      "Network density: 0.0502\n",
      "Most connected CIK: ENTERGY LOUISIANA, LLC with 5 connections\n",
      "Number of clusters: 18\n",
      "Sizes of the 5 largest clusters:\n",
      "  Cluster 1: 6 CIKs\n",
      "  Cluster 2: 6 CIKs\n",
      "  Cluster 3: 5 CIKs\n",
      "  Cluster 4: 3 CIKs\n",
      "  Cluster 5: 3 CIKs\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import networkx as nx\n",
    "from collections import defaultdict\n",
    "\n",
    "connections = defaultdict(set)\n",
    "\n",
    "# Populate the connections dictionary\n",
    "for _, group in joint_filers_tickers.groupby('accession_number'):\n",
    "    titles = group['title'].unique()\n",
    "    for title in titles:\n",
    "        connections[title].update(set(titles) - {title})\n",
    "\n",
    "# Create a graph\n",
    "G = nx.Graph(connections)\n",
    "\n",
    "# Draw the graph\n",
    "plt.figure(figsize=(20, 16))  # Increased figure size\n",
    "pos = nx.kamada_kawai_layout(G)  # Using Kamada-Kawai layout for better spacing\n",
    "nx.draw(G, pos, node_color='lightblue', \n",
    "        node_size=1000,  # Reduced node size\n",
    "        with_labels=False)  # We'll add labels separately\n",
    "\n",
    "# Add node labels\n",
    "for node, (x, y) in pos.items():\n",
    "    plt.text(x, y, node, fontsize=8, ha='center', va='center')\n",
    "\n",
    "# Add a title\n",
    "plt.title(\"Companies that filed joint 8-Ks\", fontsize=16)\n",
    "\n",
    "# Show the plot\n",
    "plt.axis('off')\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# Print some basic network statistics\n",
    "print(f\"Number of nodes (CIKs): {G.number_of_nodes()}\")\n",
    "print(f\"Number of edges (connections): {G.number_of_edges()}\")\n",
    "print(f\"Network density: {nx.density(G):.4f}\")\n",
    "\n",
    "# Find the CIK with the most connections\n",
    "most_connected = max(G.degree, key=lambda x: x[1])\n",
    "print(f\"Most connected CIK: {most_connected[0]} with {most_connected[1]} connections\")\n",
    "\n",
    "# Identify clusters (connected components)\n",
    "clusters = list(nx.connected_components(G))\n",
    "print(f\"Number of clusters: {len(clusters)}\")\n",
    "print(\"Sizes of the 5 largest clusters:\")\n",
    "for i, cluster in enumerate(sorted(clusters, key=len, reverse=True)[:5], 1):\n",
    "    print(f\"  Cluster {i}: {len(cluster)} CIKs\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
