-
-
Notifications
You must be signed in to change notification settings - Fork 2
/
linkscraper.yml
22 lines (18 loc) · 1.19 KB
/
linkscraper.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
general:
enable_logs: true # Boolean (true/false)
enable_plugins: true # Boolean (true/false)
default_user_agent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36' # String
storage:
logs: './logs' # String
downloads: './data/downloads' # String
screenshots: './data/screenshots' # String
format_dates:
datetime: '%Y-%m-%d %H:%M:%S' # String
today_datetime: '%a, %b %d %Y - %I:%M:%S %p' # String
dataset:
list_http_status: 'https://gist.githubusercontent.com/kremilly/31aed3c28577f78bcaace2a377f8aa17/raw/4e765d040b739f0d7bc60ef8c2c8f37c17eae81c/http-status-code.json' # String
list_countries: 'https://gist.githubusercontent.com/kremilly/c468fb230d6fcf97de827e37f91f2f6c/raw/3e037cefa50d0381956e862de478c5e5cce758ab/countries.json' # String
list_languages: 'https://gist.githubusercontent.com/kremilly/fd5e5dd45d3480a8da57d56218cecd1e/raw/221c38e4a7e83e2bb9bab92cd8101c9c9adebaaf/languages.json' # String
list_allowed_extensions: 'https://gist.githubusercontent.com/kremilly/5ce0a3c85bc7f6c5b2c49a52aa8ecd36/raw/551cc92e5f3a62f2c7e631add0c5ccb19312a6df/allowed-extensions.json' # String
advanced:
max_workers: 5 # Integer (unsigned)