* Make search faster with multi-thread implementation * Show error messages per indexer * Number of threads is configurable in jackett.json config file Based on #222 Thank you @galeksandrp and @WojtekKowaluk
This commit is contained in:
Diego Heras
GitHub
parent
3508f23400
commit
6677f9a8c0
|
|
@ -1,7 +1,8 @@
|
|||
#VERSION: 3.5
|
||||
#VERSION: 4.0
|
||||
# AUTHORS: Diego de las Heras (ngosang@hotmail.es)
|
||||
# CONTRIBUTORS: ukharley
|
||||
# hannsen (github.com/hannsen)
|
||||
# Alexander Georgievskiy <galeksandrp@gmail.com>
|
||||
|
||||
import json
|
||||
import os
|
||||
|
|
@ -9,6 +10,8 @@ import xml.etree.ElementTree
|
|||
from urllib.parse import urlencode, unquote
|
||||
from urllib import request as urllib_request
|
||||
from http.cookiejar import CookieJar
|
||||
from multiprocessing.dummy import Pool
|
||||
from threading import Lock
|
||||
|
||||
from novaprinter import prettyPrinter
|
||||
from helpers import download_file
|
||||
|
|
@ -20,9 +23,11 @@ CONFIG_FILE = 'jackett.json'
|
|||
CONFIG_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), CONFIG_FILE)
|
||||
CONFIG_DATA = {
|
||||
'api_key': 'YOUR_API_KEY_HERE', # jackett api
|
||||
'tracker_first': False, # (False/True) add tracker name to beginning of search result
|
||||
'url': 'http://127.0.0.1:9117', # jackett url
|
||||
'tracker_first': False, # (False/True) add tracker name to beginning of search result
|
||||
'thread_count': 20, # number of threads to use for http requests
|
||||
}
|
||||
PRINTER_THREAD_LOCK = Lock()
|
||||
|
||||
|
||||
def load_configuration():
|
||||
|
|
@ -32,17 +37,27 @@ def load_configuration():
|
|||
with open(CONFIG_PATH) as f:
|
||||
CONFIG_DATA = json.load(f)
|
||||
except ValueError:
|
||||
# if file exists but it's malformed we load add a flag
|
||||
# if file exists, but it's malformed we load add a flag
|
||||
CONFIG_DATA['malformed'] = True
|
||||
except Exception:
|
||||
# if file doesn't exist, we create it
|
||||
with open(CONFIG_PATH, 'w') as f:
|
||||
f.write(json.dumps(CONFIG_DATA, indent=4, sort_keys=True))
|
||||
save_configuration()
|
||||
|
||||
# do some checks
|
||||
if any(item not in CONFIG_DATA for item in ['api_key', 'tracker_first', 'url']):
|
||||
CONFIG_DATA['malformed'] = True
|
||||
|
||||
# add missing keys
|
||||
if 'thread_count' not in CONFIG_DATA:
|
||||
CONFIG_DATA['thread_count'] = 20
|
||||
save_configuration()
|
||||
|
||||
|
||||
def save_configuration():
|
||||
global CONFIG_PATH, CONFIG_DATA
|
||||
with open(CONFIG_PATH, 'w') as f:
|
||||
f.write(json.dumps(CONFIG_DATA, indent=4, sort_keys=True))
|
||||
|
||||
|
||||
load_configuration()
|
||||
###############################################################################
|
||||
|
|
@ -52,6 +67,7 @@ class jackett(object):
|
|||
name = 'Jackett'
|
||||
url = CONFIG_DATA['url'] if CONFIG_DATA['url'][-1] != '/' else CONFIG_DATA['url'][:-1]
|
||||
api_key = CONFIG_DATA['api_key']
|
||||
thread_count = CONFIG_DATA['thread_count']
|
||||
supported_categories = {
|
||||
'all': None,
|
||||
'anime': ['5070'],
|
||||
|
|
@ -87,6 +103,37 @@ class jackett(object):
|
|||
self.handle_error("api key error", what)
|
||||
return
|
||||
|
||||
# search in Jackett API
|
||||
if self.thread_count > 1:
|
||||
args = []
|
||||
indexers = self.get_jackett_indexers(what)
|
||||
for indexer in indexers:
|
||||
args.append((what, category, indexer))
|
||||
with Pool(min(len(indexers), self.thread_count)) as pool:
|
||||
pool.starmap(self.search_jackett_indexer, args)
|
||||
else:
|
||||
self.search_jackett_indexer(what, category, 'all')
|
||||
|
||||
def get_jackett_indexers(self, what):
|
||||
params = [
|
||||
('apikey', self.api_key),
|
||||
('t', 'indexers'),
|
||||
('configured', 'true')
|
||||
]
|
||||
params = urlencode(params)
|
||||
jacket_url = self.url + "/api/v2.0/indexers/all/results/torznab/api?%s" % params
|
||||
response = self.get_response(jacket_url)
|
||||
if response is None:
|
||||
self.handle_error("connection error getting indexer list", what)
|
||||
return
|
||||
# process results
|
||||
response_xml = xml.etree.ElementTree.fromstring(response)
|
||||
indexers = []
|
||||
for indexer in response_xml.findall('indexer'):
|
||||
indexers.append(indexer.attrib['id'])
|
||||
return indexers
|
||||
|
||||
def search_jackett_indexer(self, what, category, indexer_id):
|
||||
# prepare jackett url
|
||||
params = [
|
||||
('apikey', self.api_key),
|
||||
|
|
@ -95,12 +142,11 @@ class jackett(object):
|
|||
if category is not None:
|
||||
params.append(('cat', ','.join(category)))
|
||||
params = urlencode(params)
|
||||
jacket_url = self.url + "/api/v2.0/indexers/all/results/torznab/api?%s" % params
|
||||
jacket_url = self.url + "/api/v2.0/indexers/" + indexer_id + "/results/torznab/api?%s" % params # noqa
|
||||
response = self.get_response(jacket_url)
|
||||
if response is None:
|
||||
self.handle_error("connection error", what)
|
||||
self.handle_error("connection error for indexer: " + indexer_id, what)
|
||||
return
|
||||
|
||||
# process search results
|
||||
response_xml = xml.etree.ElementTree.fromstring(response)
|
||||
for result in response_xml.find('channel').findall('item'):
|
||||
|
|
@ -151,18 +197,11 @@ class jackett(object):
|
|||
# note: engine_url can't be changed, torrent download stops working
|
||||
res['engine_url'] = self.url
|
||||
|
||||
prettyPrinter(self.escape_pipe(res))
|
||||
self.pretty_printer_thread_safe(res)
|
||||
|
||||
def generate_xpath(self, tag):
|
||||
return './{http://torznab.com/schemas/2015/feed}attr[@name="%s"]' % tag
|
||||
|
||||
# Safety measure until it's fixed in prettyPrinter
|
||||
def escape_pipe(self, dictionary):
|
||||
for key in dictionary.keys():
|
||||
if isinstance(dictionary[key], str):
|
||||
dictionary[key] = dictionary[key].replace('|', '%7C')
|
||||
return dictionary
|
||||
|
||||
def get_response(self, query):
|
||||
response = None
|
||||
try:
|
||||
|
|
@ -181,7 +220,7 @@ class jackett(object):
|
|||
def handle_error(self, error_msg, what):
|
||||
# we need to print the search text to be displayed in qBittorrent when
|
||||
# 'Torrent names only' is enabled
|
||||
prettyPrinter({
|
||||
self.pretty_printer_thread_safe({
|
||||
'seeds': -1,
|
||||
'size': -1,
|
||||
'leech': -1,
|
||||
|
|
@ -191,6 +230,18 @@ class jackett(object):
|
|||
'name': "Jackett: %s! Right-click this row and select 'Open description page' to open help. Configuration file: '%s' Search: '%s'" % (error_msg, CONFIG_PATH, what) # noqa
|
||||
})
|
||||
|
||||
def pretty_printer_thread_safe(self, dictionary):
|
||||
global PRINTER_THREAD_LOCK
|
||||
with PRINTER_THREAD_LOCK:
|
||||
prettyPrinter(self.escape_pipe(dictionary))
|
||||
|
||||
def escape_pipe(self, dictionary):
|
||||
# Safety measure until it's fixed in prettyPrinter
|
||||
for key in dictionary.keys():
|
||||
if isinstance(dictionary[key], str):
|
||||
dictionary[key] = dictionary[key].replace('|', '%7C')
|
||||
return dictionary
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
jackett_se = jackett()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
eztv: 1.14
|
||||
jackett: 3.5
|
||||
jackett: 4.0
|
||||
limetorrents: 4.7
|
||||
piratebay: 3.2
|
||||
rarbg: 2.14
|
||||
|
|
|
|||
Loading…
Reference in New Issue