Adds sleep as an argument

This commit is contained in:
J 2019-09-18 19:14:03 +02:00
parent 50d4ad6e93
commit 7438e0f5a7
3 changed files with 5 additions and 36 deletions

View File

@ -3,16 +3,20 @@ import argparse
from src.librecatastro.scrapping.scrapper_html import ScrapperHTML
from src.librecatastro.scrapping.scrapper_xml import ScrapperXML
from src.settings import config
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Runs the Cadastro Parser')
parser.add_argument('--coords', dest='coords', default=False, action='store_true')
parser.add_argument('--filenames', action='store', nargs='+', dest='filenames', default=[])
parser.add_argument('--provinces', action='store', nargs='+', dest='provinces', default=[])
parser.add_argument('--sleep', action='store', dest='sleep', type=int, default=5)
args = parser.parse_args(sys.argv[1:])
config['sleep_time'] = args.sleep
if args.coords:
print(args.filenames)
ScrapperHTML.scrap_all_coordinates_files(args.filenames)
else:
ScrapperXML.scrap_all_addresses(args.provinces)

View File

@ -1,32 +0,0 @@
import json
from collections import namedtuple
from src.settings import config
from src.utils.cadastro_logger import CadastroLogger
'''Logger'''
logger = CadastroLogger(__name__).logger
class GeoBoundingBox:
def __init__(self, data):
self.data = json.loads(data, object_hook=lambda d: namedtuple('X', d.keys())(*d.values()))
def get_coordinates_tuple(self):
return GeoBoundingBox.get_bb_from_file_static(self.data)
@staticmethod
def get_bb_from_file_static(data):
location = data.geo_bounding_box.location
return int(location.top_left.lon * config['scale']), int(location.bottom_right.lon * config['scale']), int(location.bottom_right.lat * config['scale']), int(location.top_left.lat * config['scale'])
@staticmethod
def get_bb_from_file(file):
f = open(file, "r")
content = f.read()
try:
data = json.loads(content, object_hook=lambda d: namedtuple('X', d.keys())(*d.values()))
return GeoBoundingBox.get_bb_from_file_static(data)
except:
logger.error("{} is not formatted properly. Please take a look at the examples.".format(file))
return None

View File

@ -1,11 +1,8 @@
import os
import unittest
from shapely.geometry import Point
from src.librecatastro.domain.geometry.geo_polygon import GeoPolygon
from src.librecatastro.scrapping.scrapper_html import ScrapperHTML
from src.librecatastro.domain.geometry.geo_bounding_box import GeoBoundingBox
from src.settings import config
from src.utils.elasticsearch_utils import ElasticSearchUtils