http: make RequetsClient.file use streaming
This commit is contained in:
parent
48f0178405
commit
096dfd159b
|
@ -97,7 +97,7 @@ class HttpClient(object):
|
||||||
request.remove_header('host')
|
request.remove_header('host')
|
||||||
|
|
||||||
|
|
||||||
def __build_request(self, url, data=None, headers={}, method='GET'):
|
def __build_request(self, url, data=None, headers={}, method='GET', stream=False):
|
||||||
new_headers = self.headers.copy()
|
new_headers = self.headers.copy()
|
||||||
new_headers.update(headers)
|
new_headers.update(headers)
|
||||||
|
|
||||||
|
@ -112,7 +112,7 @@ class HttpClient(object):
|
||||||
if isinstance(data, str):
|
if isinstance(data, str):
|
||||||
data = data.encode('UTF-8')
|
data = data.encode('UTF-8')
|
||||||
|
|
||||||
request = Request(url, data=data, headers=parsed_headers, method=method)
|
request = Request(url, data=data, headers=parsed_headers, method=method, stream=stream)
|
||||||
|
|
||||||
if self.proxy.enabled:
|
if self.proxy.enabled:
|
||||||
request.set_proxy(f'{self.proxy.host}:{self.proxy.port}', self.proxy.ptype)
|
request.set_proxy(f'{self.proxy.host}:{self.proxy.port}', self.proxy.ptype)
|
||||||
|
@ -136,21 +136,31 @@ class HttpClient(object):
|
||||||
return HttpResponse(response)
|
return HttpResponse(response)
|
||||||
|
|
||||||
|
|
||||||
def file(self, url, filepath, *args, filename=None, **kwargs):
|
def file(self, url, filepath, *args, filename=None, size=2048, **kwargs):
|
||||||
resp = self.request(url, *args, **kwargs)
|
path = Path(filepath)
|
||||||
|
filepath = path.join(filename if filename else Path(url).name)
|
||||||
|
|
||||||
|
if filepath.exists():
|
||||||
|
kwargs['headers']['range'] = f'bytes={filepath.size}'
|
||||||
|
|
||||||
|
resp = self.request(url, *args, stream=True, **kwargs)
|
||||||
|
|
||||||
|
if not resp.headers.get('content-length'):
|
||||||
|
logging.verbose('File already downloaded fully')
|
||||||
|
return True
|
||||||
|
|
||||||
if resp.status != 200:
|
if resp.status != 200:
|
||||||
logging.error(f'Failed to download {url}:', resp.status, resp.body)
|
logging.error(f'Failed to download {url}:', resp.status, resp.body)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
path = Path(filepath)
|
|
||||||
|
|
||||||
if not path.exists():
|
if not path.exists():
|
||||||
logging.error('Path does not exist:', path)
|
logging.error('Path does not exist:', path)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
with path.join(filename).open('wb') as fd:
|
with filepath.open('ab') as fd:
|
||||||
fd.write(resp.body)
|
for chunk in resp.chunks(size):
|
||||||
|
fd.write(chunk)
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -491,6 +491,10 @@ class Path(object):
|
||||||
return os.path.getmtime(self.str())
|
return os.path.getmtime(self.str())
|
||||||
|
|
||||||
|
|
||||||
|
def size(self):
|
||||||
|
return self.__path.stat().st_size
|
||||||
|
|
||||||
|
|
||||||
def link(self, path):
|
def link(self, path):
|
||||||
target = Path(path)
|
target = Path(path)
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue