{
    "cells": [
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Data matching\n",
                "---\n",
                "\n",
                "Experimenting with matching data from:\n",
                "- Global Energy Monitor (GEM)'s [Global Coal Plant Tracker](https://www.globalenergymonitor.org/coal.html)\n",
                "- USA's [CAMPD emissions data](https://campd.epa.gov/data)\n",
                "- OSM's [cooling_tower](https://wiki.openstreetmap.org/wiki/Tag:man_made%3Dcooling_tower) tag"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Setup"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Imports"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import pandas as pd\n",
                "import geopandas as gpd\n",
                "import plotly.express as px"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from coal_emissions_monitoring.data_cleaning import (\n",
                "    load_clean_gcpt_gdf,\n",
                "    load_clean_campd_facilities_gdf,\n",
                "    load_clean_campd_emissions_df,\n",
                "    load_osm_data,\n",
                ")"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Parameters"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# show all columns in pandas\n",
                "pd.set_option(\"display.max_columns\", None)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Load data"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### GEM Global Coal Plant Tracker"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "gcpt_df = load_clean_gcpt_gdf(\"/Users/adminuser/Downloads/Global-Coal-Plant-Tracker-January-2023.xlsx\")\n",
                "gcpt_df"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### CAMPD facilities metadata"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "campd_facilities_df = load_clean_campd_facilities_gdf(\"/Users/adminuser/GitHub/ccai-ss23-ai-monitoring-tutorial/data/facility_attributes.csv\")\n",
                "campd_facilities_df"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "campd_facilities_df.capacity_mw.describe()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# find distance to the nearest facility\n",
                "for facility_id in campd_facilities_df.facility_id:\n",
                "    campd_facilities_df.loc[\n",
                "        campd_facilities_df.facility_id == facility_id,\n",
                "        \"dist_to_nearest_facility\"\n",
                "    ] = gpd.sjoin_nearest(\n",
                "        campd_facilities_df.loc[campd_facilities_df.facility_id == facility_id],\n",
                "        campd_facilities_df.loc[campd_facilities_df.facility_id != facility_id],\n",
                "        distance_col=\"dist\",\n",
                "    ).dist.min()\n",
                "campd_facilities_df.groupby(\"facility_id\").dist_to_nearest_facility.min().sort_values()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "campd_facilities_df[campd_facilities_df.year == 2023].explore()"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### CAMPD emissions data"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "campd_emissions_df = load_clean_campd_emissions_df(\"/Users/adminuser/GitHub/ccai-ss23-ai-monitoring-tutorial/data/daily_emissions_facility_aggregation.csv\")\n",
                "campd_emissions_df"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "campd_emissions_df[\"year\"] = campd_emissions_df[\"date\"].dt.year\n",
                "yearly_emissions = campd_emissions_df.groupby(\"year\").co2_mass_short_tons.mean()\n",
                "yearly_emissions"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "px.line(campd_emissions_df, x=\"date\", y=\"co2_mass_short_tons\", color=\"facility_name\")"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### OSM cooling_tower tag"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "osm_gdf = load_osm_data()\n",
                "osm_gdf"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Match data"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### CAMPD facilities metadata and emissions"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "campd_emissions_df[\"year\"] = pd.to_datetime(campd_emissions_df[\"date\"].dt.year, format=\"%Y\")\n",
                "campd_gdf = pd.merge(\n",
                "    campd_facilities_df,\n",
                "    campd_emissions_df,\n",
                "    on=[\"facility_id\", \"year\"],\n",
                "    how=\"inner\",\n",
                "    suffixes=(\"_delete\", \"\"),\n",
                ")\n",
                "campd_gdf = campd_gdf.drop(columns=[col for col in campd_gdf.columns if \"_delete\" in col])\n",
                "campd_gdf"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### CAMPD data and OSM cooling_tower tag"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "campd_ndt_gdf = gpd.sjoin_nearest(campd_gdf, osm_gdf, how=\"inner\", distance_col=\"distances\", max_distance=0.01)\n",
                "campd_ndt_gdf"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "campd_ndt_gdf.distances.describe()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "ndt_plants = campd_ndt_gdf.facility_id.nunique()\n",
                "ndt_plants"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": []
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "ccai_ss23",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.10.9"
        },
        "orig_nbformat": 4
    },
    "nbformat": 4,
    "nbformat_minor": 2
}
