esi-db-snarf/snarf.py

#!/usr/bin/env python3

import requests
import sqlite3
import time

con = sqlite3.connect('new_eden.db')
cur = con.cursor()

query_regions = "https://esi.evetech.net/v1/universe/regions/"
query_constellations = "https://esi.evetech.net/v1/universe/constellations/"
query_systems = "https://esi.evetech.net/v4/universe/systems/"
query_gates = "https://esi.evetech.net/v1/universe/stargates/"

# 11000000
regions = requests.get(query_regions).json()
for region_id in regions:
  try:
    region = requests.get(query_regions+str(region_id)).json()
  except:
    time.sleep(30)
  print("Region: "+region['name'])
  for constellation_id in region['constellations']:
    try:
      constellation = requests.get(query_constellations+str(constellation_id)).json()
    except:
      time.sleep(30)
    for system_id in constellation["systems"]:
      try:
        system = requests.get(query_systems+str(system_id)).json()
      except:
        time.sleep(30)
      # print(str(system['system_id']), system['name'], str(constellation['constellation_id']), constellation['name'], str(region['region_id']), region['name'])
      cur.execute("INSERT INTO systems VALUES (?,?,?,?,?,?)", (system['system_id'], system['name'], constellation['constellation_id'], constellation['name'], region['region_id'], region['name']))

      try:
        if len(system["stargates"]) > 0:
          for gate in system["stargates"]:
            gate_info = requests.get(query_gates+str(gate)).json()
            # print(system["system_id"], gate_info["destination"]["system_id"])
            cur.execute("INSERT INTO gates VALUES (?,?)",(system["system_id"],gate_info["destination"]["system_id"]))
      except KeyError:
        pass
    con.commit()
    time.sleep(1)
  time.sleep(10)

con.close()