{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.vectorstores import Chroma\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain.llms import OpenAI\n",
    "from langchain.chains import RetrievalQA\n",
    "from langchain.document_loaders import PyPDFLoader\n",
    "from langchain.document_loaders import DirectoryLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ninja/miniconda3/envs/secondbrain/lib/python3.9/site-packages/InstructorEmbedding/instructor.py:7: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from tqdm.autonotebook import trange\n"
     ]
    }
   ],
   "source": [
    "# InstructorEmbedding \n",
    "from InstructorEmbedding import INSTRUCTOR\n",
    "from langchain.embeddings import HuggingFaceInstructEmbeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = DirectoryLoader('docs/', glob=\"./*.pdf\", loader_cls=PyPDFLoader)\n",
    "documents = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "                                               chunk_size=1000, \n",
    "                                               chunk_overlap=200)\n",
    "\n",
    "texts = text_splitter.split_documents(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content=\"Bitcoin: A Peer-to-Peer Electronic Cash System\\nSatoshi Nakamoto\\nsatoshin@gmx.com\\nwww.bitcoin.org\\nAbstract.A purely peer-to-peer version of electronic cash would allow online\\npayments to be sent directly from one party to another without going through a\\nfinancial institution. Digital signatures provide part of the solution, but the main\\nbenefits are lost if a trusted third party is still required to prevent double-spending.\\nWe propose a solution to the double-spending problem using a peer-to-peer network.\\nThe network timestamps transactions by hashing them into an ongoing chain of\\nhash-based proof-of-work, forming a record that cannot be changed without redoing\\nthe proof-of-work. The longest chain not only serves as proof of the sequence of\\nevents witnessed, but proof that it came from the largest pool of CPU power. As\\nlong as a majority of CPU power is controlled by nodes that are not cooperating to\\nattack the network, they'll generate the longest chain and outpace attackers. The\", metadata={'source': 'docs/bitcoin.pdf', 'page': 0})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import faiss\n",
    "from langchain.vectorstores import FAISS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ninja/miniconda3/envs/secondbrain/lib/python3.9/site-packages/InstructorEmbedding/instructor.py:7: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from tqdm.autonotebook import trange\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load INSTRUCTOR_Transformer\n"
     ]
    }
   ],
   "source": [
    "from langchain.embeddings import HuggingFaceInstructEmbeddings\n",
    "\n",
    "instructor_embeddings = HuggingFaceInstructEmbeddings(model_name=\"hkunlp/instructor-xl\",\n",
    "                                                      model_kwargs={\"device\": \"cpu\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "def load_embeddings(sotre_name, path):\n",
    "    with open(f\"{path}/faiss_{sotre_name}.pkl\", \"rb\") as f:\n",
    "        VectorStore = pickle.load(f)\n",
    "    return VectorStore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[11], line 6\u001b[0m\n\u001b[1;32m      3\u001b[0m torch\u001b[39m.\u001b[39mdevice(\u001b[39m'\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m      4\u001b[0m Embedding_store_path \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mmodules/\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m----> 6\u001b[0m db_instructEmbedd \u001b[39m=\u001b[39m load_embeddings(sotre_name\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39minstructEmbeddings\u001b[39;49m\u001b[39m'\u001b[39;49m, \n\u001b[1;32m      7\u001b[0m                                      path\u001b[39m=\u001b[39;49mEmbedding_store_path)\n",
      "Cell \u001b[0;32mIn[10], line 4\u001b[0m, in \u001b[0;36mload_embeddings\u001b[0;34m(sotre_name, path)\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mload_embeddings\u001b[39m(sotre_name, path):\n\u001b[1;32m      3\u001b[0m     \u001b[39mwith\u001b[39;00m \u001b[39mopen\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mpath\u001b[39m}\u001b[39;00m\u001b[39m/faiss_\u001b[39m\u001b[39m{\u001b[39;00msotre_name\u001b[39m}\u001b[39;00m\u001b[39m.pkl\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mrb\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39mas\u001b[39;00m f:\n\u001b[0;32m----> 4\u001b[0m         VectorStore \u001b[39m=\u001b[39m pickle\u001b[39m.\u001b[39;49mload(f)\n\u001b[1;32m      5\u001b[0m     \u001b[39mreturn\u001b[39;00m VectorStore\n",
      "File \u001b[0;32m~/miniconda3/envs/secondbrain/lib/python3.9/site-packages/torch/storage.py:241\u001b[0m, in \u001b[0;36m_load_from_bytes\u001b[0;34m(b)\u001b[0m\n\u001b[1;32m    240\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_load_from_bytes\u001b[39m(b):\n\u001b[0;32m--> 241\u001b[0m     \u001b[39mreturn\u001b[39;00m torch\u001b[39m.\u001b[39;49mload(io\u001b[39m.\u001b[39;49mBytesIO(b))\n",
      "File \u001b[0;32m~/miniconda3/envs/secondbrain/lib/python3.9/site-packages/torch/serialization.py:815\u001b[0m, in \u001b[0;36mload\u001b[0;34m(f, map_location, pickle_module, weights_only, **pickle_load_args)\u001b[0m\n\u001b[1;32m    813\u001b[0m     \u001b[39mexcept\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    814\u001b[0m         \u001b[39mraise\u001b[39;00m pickle\u001b[39m.\u001b[39mUnpicklingError(UNSAFE_MESSAGE \u001b[39m+\u001b[39m \u001b[39mstr\u001b[39m(e)) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m--> 815\u001b[0m \u001b[39mreturn\u001b[39;00m _legacy_load(opened_file, map_location, pickle_module, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mpickle_load_args)\n",
      "File \u001b[0;32m~/miniconda3/envs/secondbrain/lib/python3.9/site-packages/torch/serialization.py:1043\u001b[0m, in \u001b[0;36m_legacy_load\u001b[0;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[1;32m   1041\u001b[0m unpickler \u001b[39m=\u001b[39m UnpicklerWrapper(f, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mpickle_load_args)\n\u001b[1;32m   1042\u001b[0m unpickler\u001b[39m.\u001b[39mpersistent_load \u001b[39m=\u001b[39m persistent_load\n\u001b[0;32m-> 1043\u001b[0m result \u001b[39m=\u001b[39m unpickler\u001b[39m.\u001b[39;49mload()\n\u001b[1;32m   1045\u001b[0m deserialized_storage_keys \u001b[39m=\u001b[39m pickle_module\u001b[39m.\u001b[39mload(f, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mpickle_load_args)\n\u001b[1;32m   1047\u001b[0m offset \u001b[39m=\u001b[39m f\u001b[39m.\u001b[39mtell() \u001b[39mif\u001b[39;00m f_should_read_directly \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/secondbrain/lib/python3.9/site-packages/torch/serialization.py:980\u001b[0m, in \u001b[0;36m_legacy_load.<locals>.persistent_load\u001b[0;34m(saved_id)\u001b[0m\n\u001b[1;32m    976\u001b[0m     obj\u001b[39m.\u001b[39m_torch_load_uninitialized \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m    977\u001b[0m     \u001b[39m# TODO: Once we decide to break serialization FC, we can\u001b[39;00m\n\u001b[1;32m    978\u001b[0m     \u001b[39m# stop wrapping with TypedStorage\u001b[39;00m\n\u001b[1;32m    979\u001b[0m     typed_storage \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mstorage\u001b[39m.\u001b[39mTypedStorage(\n\u001b[0;32m--> 980\u001b[0m         wrap_storage\u001b[39m=\u001b[39mrestore_location(obj, location),\n\u001b[1;32m    981\u001b[0m         dtype\u001b[39m=\u001b[39mdtype,\n\u001b[1;32m    982\u001b[0m         _internal\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m    983\u001b[0m     deserialized_objects[root_key] \u001b[39m=\u001b[39m typed_storage\n\u001b[1;32m    984\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/secondbrain/lib/python3.9/site-packages/torch/serialization.py:217\u001b[0m, in \u001b[0;36mdefault_restore_location\u001b[0;34m(storage, location)\u001b[0m\n\u001b[1;32m    215\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdefault_restore_location\u001b[39m(storage, location):\n\u001b[1;32m    216\u001b[0m     \u001b[39mfor\u001b[39;00m _, _, fn \u001b[39min\u001b[39;00m _package_registry:\n\u001b[0;32m--> 217\u001b[0m         result \u001b[39m=\u001b[39m fn(storage, location)\n\u001b[1;32m    218\u001b[0m         \u001b[39mif\u001b[39;00m result \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    219\u001b[0m             \u001b[39mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/miniconda3/envs/secondbrain/lib/python3.9/site-packages/torch/serialization.py:182\u001b[0m, in \u001b[0;36m_cuda_deserialize\u001b[0;34m(obj, location)\u001b[0m\n\u001b[1;32m    180\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_cuda_deserialize\u001b[39m(obj, location):\n\u001b[1;32m    181\u001b[0m     \u001b[39mif\u001b[39;00m location\u001b[39m.\u001b[39mstartswith(\u001b[39m'\u001b[39m\u001b[39mcuda\u001b[39m\u001b[39m'\u001b[39m):\n\u001b[0;32m--> 182\u001b[0m         device \u001b[39m=\u001b[39m validate_cuda_device(location)\n\u001b[1;32m    183\u001b[0m         \u001b[39mif\u001b[39;00m \u001b[39mgetattr\u001b[39m(obj, \u001b[39m\"\u001b[39m\u001b[39m_torch_load_uninitialized\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mFalse\u001b[39;00m):\n\u001b[1;32m    184\u001b[0m             \u001b[39mwith\u001b[39;00m torch\u001b[39m.\u001b[39mcuda\u001b[39m.\u001b[39mdevice(device):\n",
      "File \u001b[0;32m~/miniconda3/envs/secondbrain/lib/python3.9/site-packages/torch/serialization.py:166\u001b[0m, in \u001b[0;36mvalidate_cuda_device\u001b[0;34m(location)\u001b[0m\n\u001b[1;32m    163\u001b[0m device \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mcuda\u001b[39m.\u001b[39m_utils\u001b[39m.\u001b[39m_get_device_index(location, \u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m    165\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m torch\u001b[39m.\u001b[39mcuda\u001b[39m.\u001b[39mis_available():\n\u001b[0;32m--> 166\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\u001b[39m'\u001b[39m\u001b[39mAttempting to deserialize object on a CUDA \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m    167\u001b[0m                        \u001b[39m'\u001b[39m\u001b[39mdevice but torch.cuda.is_available() is False. \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m    168\u001b[0m                        \u001b[39m'\u001b[39m\u001b[39mIf you are running on a CPU-only machine, \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m    169\u001b[0m                        \u001b[39m'\u001b[39m\u001b[39mplease use torch.load with map_location=torch.device(\u001b[39m\u001b[39m\\'\u001b[39;00m\u001b[39mcpu\u001b[39m\u001b[39m\\'\u001b[39;00m\u001b[39m) \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m    170\u001b[0m                        \u001b[39m'\u001b[39m\u001b[39mto map your storages to the CPU.\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m    171\u001b[0m device_count \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mcuda\u001b[39m.\u001b[39mdevice_count()\n\u001b[1;32m    172\u001b[0m \u001b[39mif\u001b[39;00m device \u001b[39m>\u001b[39m\u001b[39m=\u001b[39m device_count:\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU."
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import torch\n",
    "torch.device('cpu')\n",
    "Embedding_store_path = \"modules/\"\n",
    "\n",
    "db_instructEmbedd = load_embeddings(sotre_name='instructEmbeddings', \n",
    "                                     path=Embedding_store_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'CallbackManager' from 'langchain.callbacks.base' (/home/ninja/miniconda3/envs/secondbrain/lib/python3.9/site-packages/langchain/callbacks/base.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mlangchain\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mcallbacks\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mbase\u001b[39;00m \u001b[39mimport\u001b[39;00m CallbackManager\n",
      "\u001b[0;31mImportError\u001b[0m: cannot import name 'CallbackManager' from 'langchain.callbacks.base' (/home/ninja/miniconda3/envs/secondbrain/lib/python3.9/site-packages/langchain/callbacks/base.py)"
     ]
    }
   ],
   "source": [
    "from langchain.callbacks.base import CallbackManager"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "secondbrain",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
