Examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RITIS API for XD Segments EXAMPLE\n",
    "\n",
    "API Documentation at: https://pda-api.ritis.org/v2/docs/index.html#/\n",
    "\n",
    "### Download data for a single day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing RITIS_Downloader\n",
      "Loaded 2 XD segments\n",
      "Starting single download: start_date=2024-08-01, end_date=2024-08-02, job_name=test\n",
      "Submitting job: start_date=2024-08-01, end_date=2024-08-02, name=test\n",
      "Submitting job with UUID: 9ef4985b-f69a-47f4-8041-f307515ae9af\n",
      "Request Endpoint:\n",
      "https://pda-api.ritis.org/v2/submit/export?key=696d70fad2184e20973fbf15db374a03\n",
      "Request Data:\n",
      "{'uuid': '9ef4985b-f69a-47f4-8041-f307515ae9af', 'segments': {'type': 'xd', 'ids': ['1236893704', '1236860943']}, 'dates': [{'start': '2024-08-01', 'end': '2024-08-02'}], 'times': [{'start': '06:00:00', 'end': '19:59:00'}], 'dow': [0, 1, 2, 3, 4, 5, 6], 'dsFields': [{'id': 'inrix_xd', 'columns': ['speed', 'reference_speed', 'travel_time_minutes'], 'qualityFilter': {'thresholds': [30]}}], 'granularity': {'type': 'minutes', 'value': 15}, 'travelTimeUnits': 'seconds', 'includeIsoTzd': False}\n",
      "Job submission response: 200\n",
      "Job submitted successfully. Job ID: application_1728136985900_55860\n",
      "Job Progress: 0.0%\n",
      "Job Progress: 11.76%\n",
      "Job Progress: 34.54%\n",
      "Job Progress: 50.0%\n",
      "Job Progress: 100.0%\n",
      "Downloading and processing results for UUID: 9ef4985b-f69a-47f4-8041-f307515ae9af\n",
      "Saved parquet file: Data\\test.parquet\n",
      "Single download completed\n"
     ]
    }
   ],
   "source": [
    "api_key = open(\"API_Keys/RITIS_key.txt\", \"r\").read().strip()\n",
    "\n",
    "from RITIS_API import RITIS_Downloader\n",
    "\n",
    "updater = RITIS_Downloader(\n",
    "    api_key=api_key,\n",
    "    download_path='Data', #data will be saved in this directory\n",
    "    segments_path='sample_XD_segments.txt', #path to csv file containing segment ids\n",
    "    start_time='06:00:00', #default is '00:00:00\n",
    "    end_time='19:59:00', #default is '23:59:00'\n",
    "    bin_size=15, #Enter 5, 10, 15(default), or 60\n",
    "    units='seconds', #'seconds' or 'minutes'\n",
    "    columns=['speed', 'reference_speed', 'travel_time_minutes'], #a list including any of [\"speed\",\"average_speed\",\"reference_speed\",\"travel_time_minutes\",\"confidence_score\",\"cvalue\"]\n",
    "    confidence_score=[30], #a list including any of [10,20,30]\n",
    "    last_run_path='last_run.txt', #path for text file containing last run datetime\n",
    "    verbose=2, #0: no print, 1: print only important messages, 2: print all messages\n",
    "    sleep_time=20,\n",
    "    verify=False # Only set to False if you must\n",
    "    ) \n",
    "\n",
    "updater.single_download('2024-08-01', '2024-08-02', 'test') #start_date, end_date, job_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>xd_id</th>\n",
       "      <th>measurement_tstamp</th>\n",
       "      <th>speed</th>\n",
       "      <th>reference_speed</th>\n",
       "      <th>travel_time_seconds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1236860943</td>\n",
       "      <td>2024-08-01 06:00:00</td>\n",
       "      <td>22.37</td>\n",
       "      <td>24.0</td>\n",
       "      <td>13.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1236860943</td>\n",
       "      <td>2024-08-01 06:15:00</td>\n",
       "      <td>21.27</td>\n",
       "      <td>24.0</td>\n",
       "      <td>14.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1236860943</td>\n",
       "      <td>2024-08-01 06:30:00</td>\n",
       "      <td>24.42</td>\n",
       "      <td>24.0</td>\n",
       "      <td>12.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1236860943</td>\n",
       "      <td>2024-08-01 06:45:00</td>\n",
       "      <td>22.13</td>\n",
       "      <td>24.0</td>\n",
       "      <td>13.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1236860943</td>\n",
       "      <td>2024-08-01 07:00:00</td>\n",
       "      <td>23.09</td>\n",
       "      <td>24.0</td>\n",
       "      <td>12.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>1236893704</td>\n",
       "      <td>2024-08-01 17:15:00</td>\n",
       "      <td>55.12</td>\n",
       "      <td>51.0</td>\n",
       "      <td>36.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>1236893704</td>\n",
       "      <td>2024-08-01 17:30:00</td>\n",
       "      <td>53.08</td>\n",
       "      <td>51.0</td>\n",
       "      <td>37.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>1236893704</td>\n",
       "      <td>2024-08-01 17:45:00</td>\n",
       "      <td>55.00</td>\n",
       "      <td>51.0</td>\n",
       "      <td>36.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>1236893704</td>\n",
       "      <td>2024-08-01 18:00:00</td>\n",
       "      <td>53.44</td>\n",
       "      <td>51.0</td>\n",
       "      <td>37.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>1236893704</td>\n",
       "      <td>2024-08-01 18:15:00</td>\n",
       "      <td>52.00</td>\n",
       "      <td>51.0</td>\n",
       "      <td>38.69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>97 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         xd_id  measurement_tstamp  speed  reference_speed  \\\n",
       "0   1236860943 2024-08-01 06:00:00  22.37             24.0   \n",
       "1   1236860943 2024-08-01 06:15:00  21.27             24.0   \n",
       "2   1236860943 2024-08-01 06:30:00  24.42             24.0   \n",
       "3   1236860943 2024-08-01 06:45:00  22.13             24.0   \n",
       "4   1236860943 2024-08-01 07:00:00  23.09             24.0   \n",
       "..         ...                 ...    ...              ...   \n",
       "92  1236893704 2024-08-01 17:15:00  55.12             51.0   \n",
       "93  1236893704 2024-08-01 17:30:00  53.08             51.0   \n",
       "94  1236893704 2024-08-01 17:45:00  55.00             51.0   \n",
       "95  1236893704 2024-08-01 18:00:00  53.44             51.0   \n",
       "96  1236893704 2024-08-01 18:15:00  52.00             51.0   \n",
       "\n",
       "    travel_time_seconds  \n",
       "0                 13.36  \n",
       "1                 14.04  \n",
       "2                 12.24  \n",
       "3                 13.50  \n",
       "4                 12.94  \n",
       "..                  ...  \n",
       "92                36.50  \n",
       "93                37.90  \n",
       "94                36.58  \n",
       "95                37.65  \n",
       "96                38.69  \n",
       "\n",
       "[97 rows x 5 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read the file to confirm it worked\n",
    "import pandas as pd\n",
    "df = pd.read_parquet('Data/test.parquet')\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download data through yesterday\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing RITIS_Downloader\n",
      "Loaded 2 XD segments\n",
      "Starting daily download\n",
      "Getting dates for daily download\n",
      "Dates to process: ['2025-03-15', '2025-03-16', '2025-03-17']\n",
      "Submitting job: start_date=2025-03-15, end_date=2025-03-16, name=2025-03-15\n",
      "Submitting job with UUID: 904bb5c5-5029-4cc6-9fe9-a7ab6df05321\n",
      "Request Endpoint:\n",
      "https://pda-api.ritis.org/v2/submit/export?key=696d70fad2184e20973fbf15db374a03\n",
      "Request Data:\n",
      "{'uuid': '904bb5c5-5029-4cc6-9fe9-a7ab6df05321', 'segments': {'type': 'xd', 'ids': ['1236893704', '1236860943']}, 'dates': [{'start': '2025-03-15', 'end': '2025-03-16'}], 'times': [{'start': '06:00:00', 'end': '19:59:00'}], 'dow': [0, 1, 2, 3, 4, 5, 6], 'dsFields': [{'id': 'inrix_xd', 'columns': ['speed', 'reference_speed', 'travel_time_minutes'], 'qualityFilter': {'thresholds': [30]}}], 'granularity': {'type': 'minutes', 'value': 15}, 'travelTimeUnits': 'seconds', 'includeIsoTzd': False}\n",
      "Job submission response: 200\n",
      "Job submitted successfully. Job ID: application_1728136985900_55876\n",
      "Job Progress: 0.0%\n",
      "Job Progress: 0.0%\n",
      "Job Progress: 0.0%\n",
      "Job Progress: 0.0%\n",
      "Job Progress: 0.0%\n",
      "Job Progress: 0.0%\n",
      "Job Progress: 5.0%\n",
      "Job Progress: 5.43%\n",
      "Job Progress: 8.7%\n",
      "Job Progress: 13.07%\n",
      "Job Progress: 19.45%\n",
      "Job Progress: 26.94%\n",
      "Job Progress: 41.6%\n",
      "Job Progress: 48.8%\n",
      "Job Progress: 100.0%\n",
      "Downloading and processing results for UUID: 904bb5c5-5029-4cc6-9fe9-a7ab6df05321\n",
      "Saved parquet file: Data\\2025-03-15.parquet\n",
      "Submitting job: start_date=2025-03-16, end_date=2025-03-17, name=2025-03-16\n",
      "Submitting job with UUID: 7a485485-6d4d-4bb1-8f11-da788a3482c9\n",
      "Request Endpoint:\n",
      "https://pda-api.ritis.org/v2/submit/export?key=696d70fad2184e20973fbf15db374a03\n",
      "Request Data:\n",
      "{'uuid': '7a485485-6d4d-4bb1-8f11-da788a3482c9', 'segments': {'type': 'xd', 'ids': ['1236893704', '1236860943']}, 'dates': [{'start': '2025-03-16', 'end': '2025-03-17'}], 'times': [{'start': '06:00:00', 'end': '19:59:00'}], 'dow': [0, 1, 2, 3, 4, 5, 6], 'dsFields': [{'id': 'inrix_xd', 'columns': ['speed', 'reference_speed', 'travel_time_minutes'], 'qualityFilter': {'thresholds': [30]}}], 'granularity': {'type': 'minutes', 'value': 15}, 'travelTimeUnits': 'seconds', 'includeIsoTzd': False}\n",
      "Job submission response: 200\n",
      "Job submitted successfully. Job ID: application_1728136985900_55881\n",
      "Job Progress: 0.0%\n",
      "Job Progress: 16.05%\n"
     ]
    }
   ],
   "source": [
    "#api_key = open(\"key.txt\", \"r\").read().strip()\n",
    "api_key = open(\"API_Keys/RITIS_key.txt\", \"r\").read().strip()\n",
    "\n",
    "from RITIS_API import RITIS_Downloader\n",
    "\n",
    "updater = RITIS_Downloader(\n",
    "    api_key=api_key,\n",
    "    download_path='Data', #data will be saved in this directory\n",
    "    segments_path='sample_XD_segments.txt', #path to csv file containing segment ids\n",
    "    start_time='06:00:00', #default is '00:00:00\n",
    "    end_time='19:59:00', #default is '23:59:00'\n",
    "    bin_size=15, #Enter 5, 10, 15(default), or 60\n",
    "    units='seconds', #'seconds' or 'minutes'\n",
    "    columns=['speed', 'reference_speed', 'travel_time_minutes'], #a list including any of [\"speed\",\"average_speed\",\"reference_speed\",\"travel_time_minutes\",\"confidence_score\",\"cvalue\"]\n",
    "    confidence_score=[30], #a list including any of [10,20,30]\n",
    "    last_run_path='last_run.txt', #path for text file containing last run datetime\n",
    "    verbose=2, #0: no print, 1: print only important messages, 2: print all messages\n",
    "    sleep_time=60,\n",
    "    verify=False # Only set to false if you must\n",
    "    ) \n",
    "\n",
    "updater.daily_download()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# INRIX Real-Time Speed Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INRIX_Downloader: Initializing INRIX_Downloader\n",
      "INRIX_Downloader: Reading file: API_Keys/INRIX_appId.txt\n",
      "INRIX_Downloader: Reading file: API_Keys/INRIX_hashToken.txt\n",
      "INRIX_Downloader: Reading segments from: sample_XD_segments.txt\n",
      "INRIX_Downloader: Loaded 2 segments\n",
      "INRIX_Downloader: Loading or refreshing token\n",
      "INRIX_Downloader: Token loaded from file\n",
      "INRIX_Downloader: Current time (UTC): 2025-03-18 16:22:47.445217+00:00, Token expiry: 2024-10-02 00:33:20.004257+00:00\n",
      "INRIX_Downloader: Time until token expiry: -168 days, 8:10:32.559040\n",
      "INRIX_Downloader: Token needs refresh, getting new token\n",
      "INRIX_Downloader: Getting new token\n",
      "INRIX_Downloader: New token obtained and saved\n",
      "INRIX_Downloader: Getting speed data\n",
      "INRIX_Downloader: Requesting data for segments 1 to 2\n",
      "INRIX_Downloader: Received data for 2 segments\n",
      "INRIX_Downloader: Total segments data received: 2\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>code</th>\n",
       "      <th>type</th>\n",
       "      <th>speed</th>\n",
       "      <th>average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1236893704</td>\n",
       "      <td>XDS</td>\n",
       "      <td>49</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1236860943</td>\n",
       "      <td>XDS</td>\n",
       "      <td>21</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         code type  speed  average\n",
       "0  1236893704  XDS     49       54\n",
       "1  1236860943  XDS     21       23"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from INRIX_API import INRIX_Downloader\n",
    "\n",
    "inrix_downloader = INRIX_Downloader(\n",
    "    app_id_path='API_Keys/INRIX_appId.txt',\n",
    "    hash_token_path='API_Keys/INRIX_hashToken.txt',\n",
    "    token_path='API_Keys/INRIX_token.json',\n",
    "    segments_path='sample_XD_segments.txt',\n",
    "    verbose=2  # Set to 2 for more detailed output, 1 for less, 0 for no output\n",
    ")\n",
    "\n",
    "speed_data = inrix_downloader.get_speed_data()\n",
    "speed_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## INRIX Speed Data Example - Over 1000 segments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INRIX_Downloader: Loaded 5590 segments\n",
      "INRIX_Downloader: Getting speed data\n",
      "INRIX_Downloader: Total segments data received: 5491\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>code</th>\n",
       "      <th>type</th>\n",
       "      <th>speed</th>\n",
       "      <th>average</th>\n",
       "      <th>segmentClosed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1236893704</td>\n",
       "      <td>XDS</td>\n",
       "      <td>49.0</td>\n",
       "      <td>54</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1236860943</td>\n",
       "      <td>XDS</td>\n",
       "      <td>21.0</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1236893725</td>\n",
       "      <td>XDS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>25</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1236992050</td>\n",
       "      <td>XDS</td>\n",
       "      <td>19.0</td>\n",
       "      <td>19</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1236926528</td>\n",
       "      <td>XDS</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5486</th>\n",
       "      <td>440958961</td>\n",
       "      <td>XDS</td>\n",
       "      <td>21.0</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5487</th>\n",
       "      <td>440958964</td>\n",
       "      <td>XDS</td>\n",
       "      <td>14.0</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5488</th>\n",
       "      <td>440958968</td>\n",
       "      <td>XDS</td>\n",
       "      <td>11.0</td>\n",
       "      <td>13</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5489</th>\n",
       "      <td>440958973</td>\n",
       "      <td>XDS</td>\n",
       "      <td>31.0</td>\n",
       "      <td>26</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5490</th>\n",
       "      <td>385974270</td>\n",
       "      <td>XDS</td>\n",
       "      <td>11.0</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5491 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            code type  speed  average segmentClosed\n",
       "0     1236893704  XDS   49.0       54           NaN\n",
       "1     1236860943  XDS   21.0       23           NaN\n",
       "2     1236893725  XDS   24.0       25           NaN\n",
       "3     1236992050  XDS   19.0       19           NaN\n",
       "4     1236926528  XDS   15.0       15           NaN\n",
       "...          ...  ...    ...      ...           ...\n",
       "5486   440958961  XDS   21.0       20           NaN\n",
       "5487   440958964  XDS   14.0       16           NaN\n",
       "5488   440958968  XDS   11.0       13           NaN\n",
       "5489   440958973  XDS   31.0       26           NaN\n",
       "5490   385974270  XDS   11.0        9           NaN\n",
       "\n",
       "[5491 rows x 5 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example using lots of sements from ODOT Signals\n",
    "from INRIX_API import INRIX_Downloader\n",
    "\n",
    "inrix_downloader = INRIX_Downloader(\n",
    "    app_id_path='API_Keys/INRIX_appId.txt',\n",
    "    hash_token_path='API_Keys/INRIX_hashToken.txt',\n",
    "    token_path='API_Keys/INRIX_token.json',\n",
    "    segments_path='//scdata2/signalshar/Data_Analysis/RITIS_Massive_Data_Downloader/XD_segments.txt',\n",
    "    verbose=1  # Set to 2 for more detailed output, 1 for less, 0 for no output\n",
    ")\n",
    "\n",
    "speed_data = inrix_downloader.get_speed_data()\n",
    "speed_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have the segmentClosed column only because these segments are from an outdated map version :(\n",
    "\n",
    "Fear not, the GeometryScrapper is here to help! (next section)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# UPDATE SEGMENTS LIST & GEOMETRIES\n",
    "\n",
    "This will take a list of lat/long points and return segment geometries within a specified radius of them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(46.0242029974689, -123.91167466706), (45.9791874021872, -123.92671351604), (45.9930911891673, -123.920617724861)]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "signals_df = pd.read_parquet('Map_Data/signals.parquet') \n",
    "# extract Latitudes and Longitude into a list of tuples\n",
    "signals = list(zip(signals_df['Latitude'], signals_df['Longitude']))\n",
    "# Print first few signal lat/longs\n",
    "print(signals[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing 1874 locations in batches of 500\n",
      "Processing batch 1\n",
      "Processing batch 2\n",
      "Processing batch 3\n",
      "Processing batch 4\n",
      "Combining all batches into a single DataFrame\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>zip</th>\n",
       "      <th>country</th>\n",
       "      <th>segID</th>\n",
       "      <th>bearing</th>\n",
       "      <th>county</th>\n",
       "      <th>state</th>\n",
       "      <th>roadName</th>\n",
       "      <th>frc</th>\n",
       "      <th>startLatitude</th>\n",
       "      <th>endLongitude</th>\n",
       "      <th>startLongitude</th>\n",
       "      <th>endLatitude</th>\n",
       "      <th>miles</th>\n",
       "      <th>intersectingRoads</th>\n",
       "      <th>tmc</th>\n",
       "      <th>direction</th>\n",
       "      <th>road</th>\n",
       "      <th>firstname</th>\n",
       "      <th>roadNumber</th>\n",
       "      <th>coordinates</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>97225</td>\n",
       "      <td>USA</td>\n",
       "      <td>448842059</td>\n",
       "      <td>N</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>OR</td>\n",
       "      <td>SW SCHOLLS FERRY RD</td>\n",
       "      <td>4</td>\n",
       "      <td>45.48626</td>\n",
       "      <td>-122.74823</td>\n",
       "      <td>-122.74878</td>\n",
       "      <td>45.48653</td>\n",
       "      <td>0.032524</td>\n",
       "      <td>[SW OLESON RD]</td>\n",
       "      <td>448842059</td>\n",
       "      <td>N</td>\n",
       "      <td>SW SCHOLLS FERRY RD</td>\n",
       "      <td>SW OLESON RD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[-122.74878, 45.48626], [-122.74833, 45.48648...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>97051</td>\n",
       "      <td>USA</td>\n",
       "      <td>1237027426</td>\n",
       "      <td>E</td>\n",
       "      <td>COLUMBIA</td>\n",
       "      <td>OR</td>\n",
       "      <td>GABLE RD</td>\n",
       "      <td>4</td>\n",
       "      <td>45.84865</td>\n",
       "      <td>-122.81877</td>\n",
       "      <td>-122.83160</td>\n",
       "      <td>45.84729</td>\n",
       "      <td>0.669561</td>\n",
       "      <td>[OLD PORTLAND RD]</td>\n",
       "      <td>1237027426</td>\n",
       "      <td>E</td>\n",
       "      <td>GABLE RD</td>\n",
       "      <td>OLD PORTLAND RD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[-122.8316, 45.84865], [-122.83134, 45.84852]...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>97141</td>\n",
       "      <td>USA</td>\n",
       "      <td>1236850628</td>\n",
       "      <td>E</td>\n",
       "      <td>TILLAMOOK</td>\n",
       "      <td>OR</td>\n",
       "      <td>3RD ST</td>\n",
       "      <td>3</td>\n",
       "      <td>45.45632</td>\n",
       "      <td>-123.83823</td>\n",
       "      <td>-123.84315</td>\n",
       "      <td>45.45680</td>\n",
       "      <td>0.245709</td>\n",
       "      <td>[OR-6, 1ST ST]</td>\n",
       "      <td>1236850628</td>\n",
       "      <td>E</td>\n",
       "      <td>3RD ST</td>\n",
       "      <td>OR-6</td>\n",
       "      <td>6</td>\n",
       "      <td>[[-123.84315, 45.45632], [-123.84209, 45.45632...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>97223</td>\n",
       "      <td>USA</td>\n",
       "      <td>429027767</td>\n",
       "      <td>S</td>\n",
       "      <td>WASHINGTON</td>\n",
       "      <td>OR</td>\n",
       "      <td>SW 72ND AVE</td>\n",
       "      <td>4</td>\n",
       "      <td>45.42697</td>\n",
       "      <td>-122.75163</td>\n",
       "      <td>-122.75161</td>\n",
       "      <td>45.42551</td>\n",
       "      <td>0.100938</td>\n",
       "      <td>[SW HUNZIKER RD]</td>\n",
       "      <td>429027767</td>\n",
       "      <td>S</td>\n",
       "      <td>SW 72ND AVE</td>\n",
       "      <td>SW HUNZIKER RD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[-122.75161, 45.42697], [-122.75163, 45.42674...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>97209</td>\n",
       "      <td>USA</td>\n",
       "      <td>125230407</td>\n",
       "      <td>W</td>\n",
       "      <td>MULTNOMAH</td>\n",
       "      <td>OR</td>\n",
       "      <td>NW COUCH ST</td>\n",
       "      <td>5</td>\n",
       "      <td>45.52358</td>\n",
       "      <td>-122.68824</td>\n",
       "      <td>-122.68725</td>\n",
       "      <td>45.52357</td>\n",
       "      <td>0.047931</td>\n",
       "      <td>[NW 17TH AVE]</td>\n",
       "      <td>125230407</td>\n",
       "      <td>W</td>\n",
       "      <td>NW COUCH ST</td>\n",
       "      <td>NW 17TH AVE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[-122.68725, 45.52358], [-122.68795, 45.52357...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9142</th>\n",
       "      <td>97035</td>\n",
       "      <td>USA</td>\n",
       "      <td>440920611</td>\n",
       "      <td>N</td>\n",
       "      <td>CLACKAMAS</td>\n",
       "      <td>OR</td>\n",
       "      <td>BOONES FERRY RD</td>\n",
       "      <td>4</td>\n",
       "      <td>45.42448</td>\n",
       "      <td>-122.70734</td>\n",
       "      <td>-122.70788</td>\n",
       "      <td>45.42497</td>\n",
       "      <td>0.042801</td>\n",
       "      <td>[COUNTRY CLUB RD]</td>\n",
       "      <td>440920611</td>\n",
       "      <td>N</td>\n",
       "      <td>BOONES FERRY RD</td>\n",
       "      <td>COUNTRY CLUB RD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[-122.70788, 45.42448], [-122.70734, 45.42497]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9143</th>\n",
       "      <td>97070</td>\n",
       "      <td>USA</td>\n",
       "      <td>440961397</td>\n",
       "      <td>W</td>\n",
       "      <td>CLACKAMAS</td>\n",
       "      <td>OR</td>\n",
       "      <td>SW WILSONVILLE RD</td>\n",
       "      <td>4</td>\n",
       "      <td>45.30314</td>\n",
       "      <td>-122.76032</td>\n",
       "      <td>-122.75748</td>\n",
       "      <td>45.30301</td>\n",
       "      <td>0.138983</td>\n",
       "      <td>[SW TOWN CENTER LOOP, SW MEMORIAL DR]</td>\n",
       "      <td>440961397</td>\n",
       "      <td>W</td>\n",
       "      <td>SW WILSONVILLE RD</td>\n",
       "      <td>SW TOWN CENTER LOOP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[-122.75748, 45.30314], [-122.75764, 45.30314...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9144</th>\n",
       "      <td>97306</td>\n",
       "      <td>USA</td>\n",
       "      <td>1236963253</td>\n",
       "      <td>E</td>\n",
       "      <td>MARION</td>\n",
       "      <td>OR</td>\n",
       "      <td>KUEBLER BLVD SE</td>\n",
       "      <td>4</td>\n",
       "      <td>44.88285</td>\n",
       "      <td>-123.05109</td>\n",
       "      <td>-123.06122</td>\n",
       "      <td>44.88122</td>\n",
       "      <td>0.517086</td>\n",
       "      <td>[]</td>\n",
       "      <td>1236963253</td>\n",
       "      <td>E</td>\n",
       "      <td>KUEBLER BLVD SE</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[-123.06122, 44.88285], [-123.06074, 44.88289...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9145</th>\n",
       "      <td>97045</td>\n",
       "      <td>USA</td>\n",
       "      <td>448838435</td>\n",
       "      <td>N</td>\n",
       "      <td>CLACKAMAS</td>\n",
       "      <td>OR</td>\n",
       "      <td>5TH ST</td>\n",
       "      <td>4</td>\n",
       "      <td>45.34979</td>\n",
       "      <td>-122.60570</td>\n",
       "      <td>-122.60092</td>\n",
       "      <td>45.35444</td>\n",
       "      <td>0.438761</td>\n",
       "      <td>[]</td>\n",
       "      <td>448838435</td>\n",
       "      <td>N</td>\n",
       "      <td>5TH ST</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[-122.60092000000004, 45.34978999999999], [-1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9146</th>\n",
       "      <td>97035</td>\n",
       "      <td>USA</td>\n",
       "      <td>448834173</td>\n",
       "      <td>E</td>\n",
       "      <td>CLACKAMAS</td>\n",
       "      <td>OR</td>\n",
       "      <td>KRUSE WAY</td>\n",
       "      <td>4</td>\n",
       "      <td>45.41677</td>\n",
       "      <td>-122.71602</td>\n",
       "      <td>-122.71931</td>\n",
       "      <td>45.41564</td>\n",
       "      <td>0.177647</td>\n",
       "      <td>[LOWER BOONES FERRY RD]</td>\n",
       "      <td>448834173</td>\n",
       "      <td>E</td>\n",
       "      <td>KRUSE WAY</td>\n",
       "      <td>LOWER BOONES FERRY RD</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[[-122.71931, 45.41677], [-122.7185, 45.41649]...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9147 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        zip country       segID bearing      county state  \\\n",
       "0     97225     USA   448842059       N  WASHINGTON    OR   \n",
       "1     97051     USA  1237027426       E    COLUMBIA    OR   \n",
       "2     97141     USA  1236850628       E   TILLAMOOK    OR   \n",
       "3     97223     USA   429027767       S  WASHINGTON    OR   \n",
       "4     97209     USA   125230407       W   MULTNOMAH    OR   \n",
       "...     ...     ...         ...     ...         ...   ...   \n",
       "9142  97035     USA   440920611       N   CLACKAMAS    OR   \n",
       "9143  97070     USA   440961397       W   CLACKAMAS    OR   \n",
       "9144  97306     USA  1236963253       E      MARION    OR   \n",
       "9145  97045     USA   448838435       N   CLACKAMAS    OR   \n",
       "9146  97035     USA   448834173       E   CLACKAMAS    OR   \n",
       "\n",
       "                 roadName  frc  startLatitude  endLongitude  startLongitude  \\\n",
       "0     SW SCHOLLS FERRY RD    4       45.48626    -122.74823      -122.74878   \n",
       "1                GABLE RD    4       45.84865    -122.81877      -122.83160   \n",
       "2                  3RD ST    3       45.45632    -123.83823      -123.84315   \n",
       "3             SW 72ND AVE    4       45.42697    -122.75163      -122.75161   \n",
       "4             NW COUCH ST    5       45.52358    -122.68824      -122.68725   \n",
       "...                   ...  ...            ...           ...             ...   \n",
       "9142      BOONES FERRY RD    4       45.42448    -122.70734      -122.70788   \n",
       "9143    SW WILSONVILLE RD    4       45.30314    -122.76032      -122.75748   \n",
       "9144      KUEBLER BLVD SE    4       44.88285    -123.05109      -123.06122   \n",
       "9145               5TH ST    4       45.34979    -122.60570      -122.60092   \n",
       "9146            KRUSE WAY    4       45.41677    -122.71602      -122.71931   \n",
       "\n",
       "      endLatitude     miles                      intersectingRoads  \\\n",
       "0        45.48653  0.032524                         [SW OLESON RD]   \n",
       "1        45.84729  0.669561                      [OLD PORTLAND RD]   \n",
       "2        45.45680  0.245709                         [OR-6, 1ST ST]   \n",
       "3        45.42551  0.100938                       [SW HUNZIKER RD]   \n",
       "4        45.52357  0.047931                          [NW 17TH AVE]   \n",
       "...           ...       ...                                    ...   \n",
       "9142     45.42497  0.042801                      [COUNTRY CLUB RD]   \n",
       "9143     45.30301  0.138983  [SW TOWN CENTER LOOP, SW MEMORIAL DR]   \n",
       "9144     44.88122  0.517086                                     []   \n",
       "9145     45.35444  0.438761                                     []   \n",
       "9146     45.41564  0.177647                [LOWER BOONES FERRY RD]   \n",
       "\n",
       "             tmc direction                 road              firstname  \\\n",
       "0      448842059         N  SW SCHOLLS FERRY RD           SW OLESON RD   \n",
       "1     1237027426         E             GABLE RD        OLD PORTLAND RD   \n",
       "2     1236850628         E               3RD ST                   OR-6   \n",
       "3      429027767         S          SW 72ND AVE         SW HUNZIKER RD   \n",
       "4      125230407         W          NW COUCH ST            NW 17TH AVE   \n",
       "...          ...       ...                  ...                    ...   \n",
       "9142   440920611         N      BOONES FERRY RD        COUNTRY CLUB RD   \n",
       "9143   440961397         W    SW WILSONVILLE RD    SW TOWN CENTER LOOP   \n",
       "9144  1236963253         E      KUEBLER BLVD SE                   None   \n",
       "9145   448838435         N               5TH ST                   None   \n",
       "9146   448834173         E            KRUSE WAY  LOWER BOONES FERRY RD   \n",
       "\n",
       "     roadNumber                                        coordinates  \n",
       "0           NaN  [[-122.74878, 45.48626], [-122.74833, 45.48648...  \n",
       "1           NaN  [[-122.8316, 45.84865], [-122.83134, 45.84852]...  \n",
       "2             6  [[-123.84315, 45.45632], [-123.84209, 45.45632...  \n",
       "3           NaN  [[-122.75161, 45.42697], [-122.75163, 45.42674...  \n",
       "4           NaN  [[-122.68725, 45.52358], [-122.68795, 45.52357...  \n",
       "...         ...                                                ...  \n",
       "9142        NaN   [[-122.70788, 45.42448], [-122.70734, 45.42497]]  \n",
       "9143        NaN  [[-122.75748, 45.30314], [-122.75764, 45.30314...  \n",
       "9144        NaN  [[-123.06122, 44.88285], [-123.06074, 44.88289...  \n",
       "9145        NaN  [[-122.60092000000004, 45.34978999999999], [-1...  \n",
       "9146        NaN  [[-122.71931, 45.41677], [-122.7185, 45.41649]...  \n",
       "\n",
       "[9147 rows x 20 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from Geometry_Scraper import GeometryScraper\n",
    "\n",
    "# Will ask for Cookie, which you can copy from a request in your browser use dev tools\n",
    "scraper = GeometryScraper()\n",
    "geometry_data = scraper.process_locations(signals, buffer_size=50) #buffer size in yards\n",
    "# Save the data\n",
    "geometry_data.to_parquet('Map_Data/segment_geometry.parquet')\n",
    "geometry_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract SegId to list and save to txt file (comma separated)\n",
    "# This can now be used in RITIS_Downloader\n",
    "segments = list(geometry_data['segID'])\n",
    "with open('Map_Data/XD_Segments.txt', 'w') as f:\n",
    "    f.write(','.join(map(str, segments)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Troubleshooting / Getting Failed Jobs\n",
    "\n",
    "I submitted RITIS jobs but lost the terminal, so now I need to manually check the status and download the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing RITIS_Downloader\n",
      "Loaded 2 XD segments\n"
     ]
    }
   ],
   "source": [
    "api_key = open(\"API_Keys/RITIS_key.txt\", \"r\").read().strip()\n",
    "job_id = ['application_1726936226001_4313', 'application_1726936226001_4363', 'application_1726936226001_4311']\n",
    "job_uuid = ['cb12be31-2f79-4fb5-895f-83ee58e0d28b', '2678b2ab-afb9-45f3-8933-36ffe008da86', 'bad114fe-d421-4fb0-84c0-6cf3a3d459a7']\n",
    "\n",
    "from RITIS_API import RITIS_Downloader\n",
    "\n",
    "updater = RITIS_Downloader(\n",
    "    api_key=api_key,\n",
    "    download_path='Data', #data will be saved in this directory\n",
    "    segments_path='sample_XD_segments.txt', #path to csv file containing segment ids\n",
    "    verbose=2, #0: no print, 1: print only important messages, 2: print all messages\n",
    "    verify=False\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Progress: 100.0%\n",
      "Downloading and processing results for UUID: cb12be31-2f79-4fb5-895f-83ee58e0d28b\n",
      "Saved parquet file: Data\\application_1726936226001_4313.parquet\n",
      "Downloaded and processed job results for application_1726936226001_4313\n",
      "Job Progress: 100.0%\n",
      "Downloading and processing results for UUID: 2678b2ab-afb9-45f3-8933-36ffe008da86\n",
      "Saved parquet file: Data\\application_1726936226001_4363.parquet\n",
      "Downloaded and processed job results for application_1726936226001_4363\n",
      "Job Progress: 100.0%\n",
      "Downloading and processing results for UUID: bad114fe-d421-4fb0-84c0-6cf3a3d459a7\n",
      "Saved parquet file: Data\\application_1726936226001_4311.parquet\n",
      "Downloaded and processed job results for application_1726936226001_4311\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "i=0\n",
    "\n",
    "while len(job_id) > 0:\n",
    "    while i < len(job_id):\n",
    "        time.sleep(60)\n",
    "        status = updater._check_job_status(job_id[i])\n",
    "        if status == 'SUCCEEDED':\n",
    "            if updater._download_and_process_job_results(job_uuid[i], job_id[i]):\n",
    "                print(f'Downloaded and processed job results for {job_id[i]}')\n",
    "                # remove the job from the list\n",
    "                job_id.pop(i)\n",
    "                job_uuid.pop(i)\n",
    "            else:\n",
    "                print(f'Failed to download and process job results for {job_id[i]}')\n",
    "                i += 1\n",
    "        else:\n",
    "            i += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}