======================================
Download and Extract
======================================

------------------------------
Base Classes
------------------------------

.. autoclass:: nemo_curator.download.DocumentDownloader
    :members:

.. autoclass:: nemo_curator.download.DocumentIterator
    :members:

.. autoclass:: nemo_curator.download.DocumentExtractor
    :members:

.. autoclass:: nemo_curator.download.batch_download
    :members:

.. autoclass:: nemo_curator.download.download_and_extract
    :members:

.. autoclass:: nemo_curator.download.import_downloader
    :members:

.. autoclass:: nemo_curator.download.import_extractor
    :members:

.. autoclass:: nemo_curator.download.import_iterator
    :members:

------------------------------
Common Crawl
------------------------------

.. autoclass:: nemo_curator.download.download_common_crawl
    :members:

.. autoclass:: nemo_curator.download.CommonCrawlWARCDownloader
    :members:

.. autoclass:: nemo_curator.download.CommonCrawlWARCExtractor
    :members:

.. autoclass:: nemo_curator.download.CommonCrawlWARCIterator
    :members:

.. autoclass:: nemo_curator.download.CommonCrawlWARCDownloaderExtractOnly
    :members:

.. autoclass:: nemo_curator.download.JusTextExtractor
    :members:

.. autoclass:: nemo_curator.download.ResiliparseExtractor
    :members:

------------------------------
Wikipedia
------------------------------

.. autoclass:: nemo_curator.download.download_wikipedia
    :members:

.. autoclass:: nemo_curator.download.WikipediaDownloader
    :members:

.. autoclass:: nemo_curator.download.WikipediaIterator
    :members:

.. autoclass:: nemo_curator.download.WikipediaExtractor
    :members:

------------------------------
Arxiv
------------------------------

.. autoclass:: nemo_curator.download.download_arxiv
    :members:

.. autoclass:: nemo_curator.download.ArxivDownloader
    :members:

.. autoclass:: nemo_curator.download.ArxivIterator
    :members:

.. autoclass:: nemo_curator.download.ArxivExtractor
    :members:
