{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import toml\n",
    "from pathlib import Path\n",
    "\n",
    "from markdownify import markdownify as md\n",
    "from markdownify import MarkdownConverter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_md(soup, **options):\n",
    "    return MarkdownConverter(**options).convert_soup(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = toml.load(\"pages.toml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['root', 'misc', 'release_notes', 'pywgrib2_s', 'documentation', 'options'])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/problems.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/default_inv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/convert_wgrib2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/submessages.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/function.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/selecting_messages.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/types.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/bin_ieee_text_format.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/small_fast_databases.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/usage_questions.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pipes.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/for_n.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/compile_questions.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/limitations.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/speed.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/calling_wgrib2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/user_grib2tables.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/special_file_names.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/time_interpolation.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/new_grid_intro.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/gribify2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/grb2_wrt.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/wgrib2_v3.1.1_changes.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pywgrib2_s.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pywgrib2_s_mk_inv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pywgrib2_s_read_inv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pywgrib2_s_inq.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pywgrib2_s_write.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pywgrib2_s_close.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pywgrib2_s_conventions.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pywgrib2_s_install.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pywgrib2_s_wgrib2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pywgrib2_s_global_variables.html\n"
     ]
    }
   ],
   "source": [
    "for section in ['user_guide', 'release_notes', 'pywgrib2_s']:\n",
    "    for i in p[section]:\n",
    "        url = f\"{p['root']}/{i}.html\"\n",
    "        dst = Path(f\"scraped-pages/{section}/{i}.md\")\n",
    "        if not dst.parent.exists():\n",
    "            dst.parent.mkdir()\n",
    "\n",
    "        print(url)\n",
    "        page = requests.get(url)\n",
    "        soup = BeautifulSoup(page.text, \"html.parser\")\n",
    "\n",
    "        a = soup.findAll('table')\n",
    "        html = a[5].findAll(\"tr\")[3].findAll(\"td\")[0]\n",
    "\n",
    "        end = f\"\\n\\n----\\n\\n>Description: {p[section][i]}\\n\\n_Docs derived from <{url}>_\"\n",
    "\n",
    "\n",
    "        with open(dst, \"w\", encoding=\"utf-8\") as f:\n",
    "            f.write(my_md(html))\n",
    "            f.write(end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['logic', 'inventory', 'misc', 'output', 'init'])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p['options'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/else.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/elseif.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/elseif_fs.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/elseif_n.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/elseif_rec.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/elseif_reg.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/endif.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/if.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/if_delayed_error.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/if_fs.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/if_n.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/if_rec.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/if_reg.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/not_if.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/not_if_fs.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/0xSec.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/aerosol_size.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/aerosol_wavelength.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/bitmap.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/center.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/checksum.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/cluster.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/cluster_info.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_0.0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_1.0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_1.1.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_1.2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_1.3.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_1.4.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_1.5.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_1.6.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_3.0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_3.1.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_3.11.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_3.15.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_3.2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_3.20.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_3.21.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_3.6.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_3.7.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_3.8.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.1.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.10.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.11.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.11s.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.15.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.16.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.230.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.233.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.235.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.240.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.241.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.242.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.3.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.4.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.5a.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.5b.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.6.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.7.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.8.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.9.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.91.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_4.91b.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_5.0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_5.1.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_5.4.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_5.5.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_5.6.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_5.7.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/code_table_6.0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ctl_ens.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ctl_inv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/cyclic.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/disc.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/domain.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/end_ft.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/end_FT.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ens.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ext_name.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/flag_table_3.10.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/flag_table_3.3.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/flag_table_3.4.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/flag_table_3.5.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/flag_table_3.9.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ftime.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ftime1.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ftime2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ftn_api_fn0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/full_name.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/gdt.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/geolocation.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/get_byte.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/get_hex.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/get_ieee.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/get_int.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/get_int2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/grib_max_bits.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/grid.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/grid_id.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/hybrid.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ij.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ijlat.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ilat.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/JMA.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/lev.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/lev0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ll2i.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ll2ij.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/lon.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/match_inv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/Match_inv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/max.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/min.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/misc.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/MM.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/model_version_date.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/n.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/N_clusters.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/N_ens.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/nl.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/nlons.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/npts.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/number_of_coordinate_values_after_template.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/nxny.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/packing.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pds_fcst_time.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pdt.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/percent.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/post_processing.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/precision.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/print.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/prob.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/process.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/processid.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/proj4_ij2ll.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/proj4_ll2i.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/proj4_ll2ij.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/pyinv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/radius.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/range.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/reset_delayed_error.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/RT.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/s.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/S.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/s2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/scale.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/scaling.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/scan.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/Sec0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/Sec3.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/Sec4.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/Sec5.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/Sec6.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/Sec_len.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/spatial_proc.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/spectral_bands.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/spectral_bands_extname.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/start_ft.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/start_FT.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/stats.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/subcenter.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/t.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/T.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/table.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/timer.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/unix_time.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/V.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/var.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/varX.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/vector_dir.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/verf.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/vt.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/VT.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/warn_old_g2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/wave_partition.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/YY.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/inv_f77.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/last.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/last0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/nl_out.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/print_out.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/s_out.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/big_endian.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/box_ave.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/check_pdt_size.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/colon.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/config.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/count.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/end.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/error_final.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/export_lonlat.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/fix_CFSv2_fcst.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/fix_ncep.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/gctpc.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/grid_changes.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/grid_def.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/h.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/header.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/help.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ijundefine.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/import_bin.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/import_grib.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/import_grib_fs.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/import_ieee.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/import_lonlat.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/import_netcdf.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/import_text.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/limit.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/little_endian.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/mem_del.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/mem_final.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/mem_init.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/new_grid_format.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/new_grid_interpolation.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/new_grid_ipopt.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/new_grid_vectors.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/new_grid_winds.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/no_header.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/proj4.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/quit.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/read_sec.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/rewind_final.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/rewind_proc.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/rpn.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/rpn_rcl.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/rpn_sto.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/scaling_0001.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ave.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_bin_prec.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_bitmap.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_byte.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_center.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_date.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ensm_derived_fcst.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ens_num.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_flag_table_3.3.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_flag_table_3.4.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ftime.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ftime1.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ftime2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_gds.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_grib_max_bits.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_grib_type.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_hex.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ieee.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ijval.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_int.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_int2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ival.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_lev.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_metadata.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_metadata_str.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_pdt.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_percentile.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_prob.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_radius.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_scaling.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_sec_size.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ts_dates.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_var.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/start_timer.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/status.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/submsg.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/sys.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/text_col.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/text_fmt.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/udf.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/udf_arg.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/undefine.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/undefine_val.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/v.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/v0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/v1.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/v2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/v97.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/v98.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/v99.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/version.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/AAIG.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/AAIGlong.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ave.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ave0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ave_var.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/bin.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/cress_lola.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/csv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/csv_long.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/cubeface2global.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ens_processing.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ens_qc.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/fcst_ave.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/fcst_ave0.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/fi.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/grib.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/GRIB.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/grib_ieee.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/grib_out.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/grib_out_irr.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/grib_out_irr2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/gribtable_used.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/gridout.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ieee.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ijbox.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ijsmall_grib.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/irr_grid.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/lola.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/merge_fcst.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/mysql.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/mysql_dump.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/mysql_speed.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ncep_norm.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ncep_uv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/netcdf.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/new_grid.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/new_grid_order.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/reduced_gaussian_grid.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/small_grib.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/spread.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/submsg_uv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/text.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/time_processing.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/tosubmsg.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/unmerge_fcst.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/wind_dir.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/wind_speed.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/wind_uv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/write_sec.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/alarm.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/append.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/crlf.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/d.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/egrep.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/egrep_v.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/eof_bin.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/eof_string.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/err_bin.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/err_string.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/fgrep.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/fgrep_v.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/fix_ncep_2.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/fix_ncep_3.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/fix_ncep_4.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/fix_undef.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/flush.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/for.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/for_n.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/g2clib.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/i.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/i_file.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/inv.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/match.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/match_fs.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/match_inv_add.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/names.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/nc3.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/nc4.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/nc_grads.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/nc_nlev.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/nc_pack.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ncpu.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/nc_table.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/nc_time.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ndate.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ndates.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/ndates_fmt.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/no_append.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/no_flush.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/no_nc_grads.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/no_nc_pack.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/no_nc_table.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/no_nc_time.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/not.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/not_fs.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/one_line.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/order.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/persistent.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/rewind_init.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ext_name.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_ext_name_chars.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_regex.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/set_version_ftime.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/tigge.html\n",
      "https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/transient.html\n"
     ]
    }
   ],
   "source": [
    "for section in p['options'].keys():\n",
    "    for name in p['options'][section]:\n",
    "        url = f\"{p['root']}/{name}.html\"\n",
    "        dst = Path(f\"./scraped-pages/options/{section}/{name}.md\")\n",
    "        if not dst.parent.exists():\n",
    "            dst.parent.mkdir(parents=True)\n",
    "\n",
    "        print(url)\n",
    "        page = requests.get(url)\n",
    "\n",
    "        if page.ok:\n",
    "\n",
    "            soup = BeautifulSoup(page.text, \"html.parser\")\n",
    "\n",
    "            a = soup.findAll('table')\n",
    "            html = a[5].findAll(\"tr\")[3].findAll(\"td\")[0]\n",
    "\n",
    "            end = f\"\\n\\n----\\n\\n>Description: {p['options'][section][name]}\\n\\n_Docs derived from <{url}>_\"\n",
    "\n",
    "            with open(dst, \"w\", encoding=\"utf-8\") as f:\n",
    "                f.write(my_md(html))\n",
    "                f.write(end)\n",
    "\n",
    "        else:\n",
    "            with open(dst, \"w\", encoding=\"utf-8\") as f:\n",
    "                f.write(f\"no docs for {name}\")\n",
    "                f.write(end)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "wgrib2-docs",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "eab4d9e2898e56734bf54442db4935b3497d056b56b2b8edebb89b953907bb40"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
