From dfe8eebeaa4a66b10a8a9a3bc9a83b48fc90ac3e Mon Sep 17 00:00:00 2001 From: coregear Date: Wed, 25 Nov 2015 10:44:47 +0800 Subject: [PATCH] _seek_to_end() performance up linux only, use "/usr/bin/wc" instead native python code, to solve startup very slow problem. --- beaver/worker/worker.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/beaver/worker/worker.py b/beaver/worker/worker.py index 25684580..74f1a0fd 100644 --- a/beaver/worker/worker.py +++ b/beaver/worker/worker.py @@ -294,9 +294,14 @@ def _seek_to_end(self): self._logger.debug("[{0}] - getting end position for {1}".format(fid, data['file'].name)) for encoding in ENCODINGS: try: - line_count = 0 - while data['file'].readline(): - line_count += 1 + #line_count = 0 + #while data['file'].readline(): + # line_count += 1 + wc = os.popen("/usr/bin/wc -lc "+data['file'].name).readline() + line_count = int(wc.split()[0]) + seek_postion = int(wc.split()[1]) + data['file'].seek(seek_postion) + self._logger.info("[{0}] - file counted {1}:{2}:{3}".format(fid, data['file'].name, line_count, seek_postion)) break except UnicodeDecodeError: self._logger.debug("[{0}] - UnicodeDecodeError raised for {1} with encoding {2}".format(fid, data['file'].name, data['encoding']))