update

lib/support/expand/ffmpeg.py

@@ -0,0 +1,425 @@
+import enum
+import os
+import platform
+import re
+import shutil
+import subprocess
+import threading
+import traceback
+from datetime import datetime
+
+from support import SupportSubprocess, SupportUtil, logger
+
+
+class SupportFfmpeg(object):
+    __instance_list = []
+    __ffmpeg_path = None
+
+    __idx = 1
+    total_callback_function = None
+    temp_path = None
+
+    @classmethod
+    def initialize(cls, __ffmpeg_path, temp_path, total_callback_function, max_pf_count=-1):
+        cls.__ffmpeg_path = __ffmpeg_path
+        cls.temp_path = temp_path
+        cls.total_callback_function = total_callback_function
+        cls.max_pf_count = max_pf_count
+
+    # retry : 재시도 횟수 
+    # max_error_packet_count : 이 숫자 초과시 중단
+    # where : 호출 모듈
+    def __init__(self, url, filename, save_path=None, max_pf_count=None, headers=None, timeout_minute=60, proxy=None, callback_id=None, callback_function=None):
+        self.__idx = str(SupportFfmpeg.__idx)
+        SupportFfmpeg.__idx += 1
+
+        self.url = url
+        self.filename = filename
+        self.save_path = save_path
+        self.max_pf_count = max_pf_count
+        self.headers = headers
+        self.timeout_minute = int(timeout_minute)
+        self.proxy = proxy
+        self.callback_id = callback_id
+        if callback_id == None:
+            self.callback_id = str(self.__idx)
+        self.callback_function = callback_function
+
+        self.temp_fullpath = os.path.join(self.temp_path, filename)
+        self.save_fullpath = os.path.join(self.save_path, filename)
+        self.thread = None
+        self.process = None
+        self.log_thread = None
+        self.status = SupportFfmpeg.Status.READY
+        self.duration = 0
+        self.duration_str = ''
+        self.current_duration = 0
+        self.percent = 0
+        #self.log = []
+        self.current_pf_count = 0
+        self.current_bitrate = ''
+        self.current_speed = ''
+        self.start_time = None
+        self.end_time = None
+        self.download_time = None
+        self.start_event = threading.Event()
+        self.exist = False
+        self.filesize = 0
+        self.filesize_str = ''
+        self.download_speed = ''
+        
+        
+        SupportFfmpeg.__instance_list.append(self)
+        if len(SupportFfmpeg.__instance_list) > 30:
+            for instance in SupportFfmpeg.__instance_list:
+                if instance.thread is None and instance.status != SupportFfmpeg.Status.READY:
+                    SupportFfmpeg.__instance_list.remove(instance)
+                    break
+                else:
+                    logger.debug('remove fail %s %s', instance.thread, self.status)
+        
+    def start(self):  
+        self.thread = threading.Thread(target=self.thread_fuction, args=())
+        self.thread.start()
+        self.start_time = datetime.now()
+        return self.get_data()
+    
+    def start_and_wait(self):
+        self.start()
+        self.thread.join(timeout=60*70)
+
+    def stop(self):
+        try:
+            self.status = SupportFfmpeg.Status.USER_STOP
+            self.kill()
+        except Exception as e:
+            logger.error(f'Exception:{str(e)}')
+            logger.error(traceback.format_exc())
+    
+    def kill(self):
+        try:
+            if self.process is not None and self.process.poll() is None:
+                import psutil
+                process = psutil.Process(self.process.pid)
+                for proc in process.children(recursive=True):
+                    proc.kill()
+                process.kill()
+        except Exception as e:
+            logger.error(f'Exception:{str(e)}')
+            logger.error(traceback.format_exc())
+            
+
+    def thread_fuction(self):
+        try:
+            if self.proxy is None:
+                if self.headers is None:
+                    command = [self.__ffmpeg_path, '-y', '-i', self.url, '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
+                else:
+                    headers_command = []
+                    for key, value in self.headers.items():
+                        if key.lower() == 'user-agent':
+                            headers_command.append('-user_agent')
+                            headers_command.append(value)
+                            pass
+                        else:
+                            headers_command.append('-headers')
+                            if platform.system() == 'Windows':
+                                headers_command.append('\'%s:%s\''%(key,value))
+                            else:
+                                headers_command.append(f'{key}:{value}')
+                    command = [self.__ffmpeg_path, '-y'] + headers_command + ['-i', self.url, '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
+            else:
+                command = [self.__ffmpeg_path, '-y', '-http_proxy', self.proxy, '-i', self.url, '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
+
+            
+            if platform.system() == 'Windows':
+                now = str(datetime.now()).replace(':', '').replace('-', '').replace(' ', '-')
+                filename = ('%s' % now) + '.mp4'
+                self.temp_fullpath = os.path.join(self.temp_path, filename)
+                command.append(self.temp_fullpath)
+            else:
+                command.append(self.temp_fullpath)
+            
+            try:
+                logger.debug(' '.join(command))
+                if os.path.exists(self.temp_fullpath):
+                    for f in SupportFfmpeg.__instance_list:
+                        if f.__idx != self.__idx and f.temp_fullpath == self.temp_fullpath and f.status in [SupportFfmpeg.Status.DOWNLOADING, SupportFfmpeg.Status.READY]:
+                            self.status = SupportFfmpeg.Status.ALREADY_DOWNLOADING
+                            return
+            except:
+                pass
+            logger.error(' '.join(command))
+            command = SupportSubprocess.command_for_windows(command)
+            self.process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf8')
+
+            self.status = SupportFfmpeg.Status.READY
+            self.log_thread = threading.Thread(target=self.log_thread_fuction, args=())
+            self.log_thread.start()
+            self.start_event.wait(timeout=60)
+            if self.log_thread is None:
+                if self.status == SupportFfmpeg.Status.READY:
+                    self.status = SupportFfmpeg.Status.ERROR
+                self.kill()
+            elif self.status == SupportFfmpeg.Status.READY:
+                self.status = SupportFfmpeg.Status.ERROR
+                self.kill()
+            else:
+                process_ret = self.process.wait(timeout=60*self.timeout_minute)
+                if process_ret is None: # timeout
+                    if self.status != SupportFfmpeg.Status.COMPLETED and self.status != SupportFfmpeg.Status.USER_STOP and self.status != SupportFfmpeg.Status.PF_STOP:
+                        self.status = SupportFfmpeg.Status.TIME_OVER
+                        self.kill()
+                else:
+                    if self.status == SupportFfmpeg.Status.DOWNLOADING:
+                        self.status = SupportFfmpeg.Status.FORCE_STOP
+            self.end_time = datetime.now()
+            self.download_time = self.end_time - self.start_time
+            try:
+                if self.status == SupportFfmpeg.Status.COMPLETED:
+                    if self.save_fullpath != self.temp_fullpath:
+                        if os.path.exists(self.save_fullpath):
+                            os.remove(self.save_fullpath)
+                        if platform.system() != 'Windows':
+                            os.system('chmod 777 "%s"' % self.temp_fullpath)
+                        shutil.move(self.temp_fullpath, self.save_fullpath)
+                        self.filesize = os.stat(self.save_fullpath).st_size
+                else:
+                    if os.path.exists(self.temp_fullpath):
+                        os.remove(self.temp_fullpath)
+            except Exception as exception:
+                logger.error('Exception:%s', exception)
+                logger.error(traceback.format_exc())
+
+            arg = {'type':'last', 'status':self.status, 'data' : self.get_data()}
+            self.send_to_listener(**arg)
+            self.process = None
+            self.thread = None
+        except Exception as e:
+            logger.error(f'Exception:{str(e)}')
+            logger.error(traceback.format_exc())
+            try:
+                self.status = SupportFfmpeg.Status.EXCEPTION
+                arg = {'type':'last', 'status':self.status, 'data' : self.get_data()}
+                self.send_to_listener(**arg)
+            except:
+                pass
+
+            
+
+    def log_thread_fuction(self):
+        with self.process.stdout:
+            for line in iter(self.process.stdout.readline, ''):
+                line = line.strip()
+                #logger.error(line)
+                try:
+                    if self.status == SupportFfmpeg.Status.READY:
+                        if line.find('Server returned 404 Not Found') != -1 or line.find('Unknown error') != -1:
+                            self.status = SupportFfmpeg.Status.WRONG_URL
+                            self.start_event.set()
+                        elif line.find('No such file or directory') != -1:
+                            self.status = SupportFfmpeg.Status.WRONG_DIRECTORY
+                            self.start_event.set()
+                        else:
+                            match = re.compile(r'Duration\:\s(\d{2})\:(\d{2})\:(\d{2})\.(\d{2})\,\sstart').search(line)
+                            if match:
+                                self.duration_str = '%s:%s:%s' % ( match.group(1), match.group(2), match.group(3))
+                                self.duration = int(match.group(4))
+                                self.duration += int(match.group(3)) * 100
+                                self.duration += int(match.group(2)) * 100 * 60
+                                self.duration += int(match.group(1)) * 100 * 60 * 60
+                                if match:
+                                    self.status = SupportFfmpeg.Status.READY
+                                    arg = {'type':'status_change', 'status':self.status, 'data' : self.get_data()}
+                                    self.send_to_listener(**arg)
+                                continue
+                            match = re.compile(r'time\=(\d{2})\:(\d{2})\:(\d{2})\.(\d{2})\sbitrate\=\s*(?P<bitrate>\d+).*?[$|\s](\s?speed\=\s*(?P<speed>.*?)x)?').search(line)
+                            if match:
+                                self.status = SupportFfmpeg.Status.DOWNLOADING
+                                arg = {'type':'status_change', 'status':self.status, 'data' : self.get_data()}
+                                self.send_to_listener(**arg)
+                                self.start_event.set()
+                    elif self.status == SupportFfmpeg.Status.DOWNLOADING:
+                        if line.find('PES packet size mismatch') != -1:
+                            self.current_pf_count += 1
+                            if self.current_pf_count > self.max_pf_count:
+                                self.status = SupportFfmpeg.Status.PF_STOP
+                                self.kill()
+                            continue
+                        if line.find('HTTP error 403 Forbidden') != -1:
+                            self.status = SupportFfmpeg.Status.HTTP_FORBIDDEN
+                            self.kill()
+                            continue
+                        match = re.compile(r'time\=(\d{2})\:(\d{2})\:(\d{2})\.(\d{2})\sbitrate\=\s*(?P<bitrate>\d+).*?[$|\s](\s?speed\=\s*(?P<speed>.*?)x)?').search(line)
+                        if match: 
+                            self.current_duration = int(match.group(4))
+                            self.current_duration += int(match.group(3)) * 100
+                            self.current_duration += int(match.group(2)) * 100 * 60
+                            self.current_duration += int(match.group(1)) * 100 * 60 * 60
+                            try:
+                                self.percent = int(self.current_duration * 100 / self.duration)
+                            except: pass
+                            self.current_bitrate = match.group('bitrate')
+                            self.current_speed = match.group('speed')
+                            self.download_time = datetime.now() - self.start_time
+                            arg = {'type':'normal', 'status':self.status, 'data' : self.get_data()}
+                            self.send_to_listener(**arg)
+                            continue
+                        match = re.compile(r'video\:\d*kB\saudio\:\d*kB').search(line)
+                        if match:
+                            self.status = SupportFfmpeg.Status.COMPLETED
+                            self.end_time = datetime.now()
+                            self.download_time = self.end_time - self.start_time
+                            self.percent = 100
+                            arg = {'type':'status_change', 'status':self.status, 'data' : self.get_data()}
+                            self.send_to_listener(**arg)
+                            continue
+                except Exception as e:
+                    logger.error(f'Exception:{str(e)}')
+                    logger.error(traceback.format_exc())
+        self.start_event.set()
+        self.log_thread = None
+
+
+    def get_data(self):
+        data = {
+            'url' : self.url,
+            'filename' : self.filename,
+            'max_pf_count' : self.max_pf_count,
+            'callback_id' : self.callback_id,
+            'temp_path' : self.temp_path,
+            'save_path' : self.save_path,
+            'temp_fullpath' : self.temp_fullpath,
+            'save_fullpath' : self.save_fullpath,
+            'status' : int(self.status),
+            'status_str' : self.status.name,
+            'status_kor' : str(self.status),
+            'duration' : self.duration,
+            'duration_str' : self.duration_str,
+            'current_duration' : self.current_duration,
+            'percent' : self.percent,
+            'current_pf_count' : self.current_pf_count,
+            'idx' : self.__idx,
+            #'log' : self.log,
+            'current_bitrate' : self.current_bitrate,
+            'current_speed' : self.current_speed,
+            'start_time' : '' if self.start_time is None else str(self.start_time).split('.')[0][5:],
+            'end_time' : '' if self.end_time is None else str(self.end_time).split('.')[0][5:],
+            'download_time' : '' if self.download_time is None else '%02d:%02d' % (self.download_time.seconds/60, self.download_time.seconds%60),
+            'exist' : os.path.exists(self.save_fullpath),
+        }                        
+        if self.status == SupportFfmpeg.Status.COMPLETED:
+            data['filesize'] = self.filesize
+            data['filesize_str'] = SupportUtil.sizeof_fmt(self.filesize)
+            if self.download_time.seconds != 0:
+                data['download_speed'] = SupportUtil.sizeof_fmt(self.filesize/self.download_time.seconds, suffix='Bytes/Second')
+            else:
+                data['download_speed'] = '0Bytes/Second'
+        return data
+
+    def send_to_listener(self, **arg):
+        if self.total_callback_function != None:
+            self.total_callback_function(**arg)
+        if self.callback_function is not None:
+            arg['callback_id'] = self.callback_id
+            self.callback_function(**arg)          
+
+
+    @classmethod
+    def stop_by_idx(cls, idx):
+        try:
+            for __instance in SupportFfmpeg.__instance_list:
+                if __instance.__idx == idx:
+                    __instance.stop()
+                    break
+        except Exception as e:
+            logger.error(f'Exception:{str(e)}')
+            logger.error(traceback.format_exc())
+
+    @classmethod
+    def get_instance_by_idx(cls, idx):
+        try:
+            for __instance in SupportFfmpeg.__instance_list:
+                if __instance.__idx == idx:
+                    return __instance
+        except Exception as e:
+            logger.error(f'Exception:{str(e)}')
+            logger.error(traceback.format_exc())
+
+    
+
+    @classmethod
+    def get_instance_by_callback_id(cls, callback_id):
+        try:
+            for __instance in SupportFfmpeg.__instance_list:
+                if __instance.callback_id == callback_id:
+                    return __instance
+        except Exception as e:
+            logger.error(f'Exception:{str(e)}')
+            logger.error(traceback.format_exc())
+        
+    @classmethod
+    def all_stop(cls):
+        try:
+            for __instance in SupportFfmpeg.__instance_list:
+                __instance.stop()
+        except Exception as e:
+            logger.error(f'Exception:{str(e)}')
+            logger.error(traceback.format_exc())
+
+    @classmethod
+    def get_list(cls):
+        return cls.__instance_list
+
+
+
+
+
+
+    class Status(enum.Enum):
+        READY = 0
+        WRONG_URL = 1
+        WRONG_DIRECTORY = 2
+        EXCEPTION = 3
+        ERROR = 4
+        HTTP_FORBIDDEN = 11
+        
+        DOWNLOADING = 5
+
+        USER_STOP = 6
+        COMPLETED = 7
+        TIME_OVER = 8
+        PF_STOP = 9
+        FORCE_STOP = 10 #강제중단
+        ALREADY_DOWNLOADING = 12 #이미 목록에 있고 다운로드중
+
+        def __int__(self):
+            return self.value
+
+        def __str__(self):
+            kor = ['준비', 'URL에러', '폴더에러', '실패(Exception)', '실패(에러)', '다운로드중', '사용자중지', '완료', '시간초과', 'PF중지', '강제중지',
+            '403에러', '임시파일이 이미 있음']
+            return kor[int(self)]
+        
+        def __repr__(self):
+            return self.name
+        
+        @staticmethod
+        def get_instance(value):
+            tmp = [ 
+                SupportFfmpeg.Status.READY, 
+                SupportFfmpeg.Status.WRONG_URL, 
+                SupportFfmpeg.Status.WRONG_DIRECTORY, 
+                SupportFfmpeg.Status.EXCEPTION, 
+                SupportFfmpeg.Status.ERROR,
+                SupportFfmpeg.Status.DOWNLOADING, 
+                SupportFfmpeg.Status.USER_STOP, 
+                SupportFfmpeg.Status.COMPLETED, 
+                SupportFfmpeg.Status.TIME_OVER, 
+                SupportFfmpeg.Status.PF_STOP, 
+                SupportFfmpeg.Status.FORCE_STOP, 
+                SupportFfmpeg.Status.HTTP_FORBIDDEN, 
+                SupportFfmpeg.Status.ALREADY_DOWNLOADING ]
+            return tmp[value]
+
+

lib/support/expand/gsheet_base.py

@@ -0,0 +1,218 @@
+import os
+import sys
+import traceback
+
+try:
+    import oauth2client
+except:
+    os.system('pip install oauth2client')
+    import oauth2client
+
+from oauth2client import tools
+from oauth2client.client import flow_from_clientsecrets
+from oauth2client.file import Storage
+
+try:
+    from apiclient.discovery import build
+except:
+    os.system('pip install google-api-python-client')
+    from apiclient.discovery import build
+
+try:
+    import time
+
+    import gspread
+    from gspread_formatting import (cellFormat, color, format_cell_range,
+                                    textFormat)
+except:
+    os.system('pip3 install gspread')
+    os.system('pip3 install gspread_formatting')
+    import gspread, time
+    from gspread_formatting  import cellFormat, textFormat, color, format_cell_range
+
+from support import d, get_logger
+
+logger = get_logger()
+
+
+
+
+        
+class GoogleSheetBase:
+        
+    current_flow = None
+
+    color_format = {
+        'green' : cellFormat(
+            backgroundColor=color(0, 1, 0), #set it to yellow
+            textFormat=textFormat(foregroundColor=color(0, 0, 0)),
+        ),
+        'yellow' : cellFormat(
+            backgroundColor=color(1, 1, 0), #set it to yellow
+            textFormat=textFormat(foregroundColor=color(0, 0, 0)),
+        ),
+        'white' : cellFormat(
+            backgroundColor=color(1, 1, 1), #set it to yellow
+            textFormat=textFormat(foregroundColor=color(0, 0, 0)),
+        )
+    }
+
+
+    def __init__(self, doc_id, credentials_filepath, tab_index, unique_header):
+        self.credentials_filepath = credentials_filepath
+        self.credentials = self.get_credentials()
+        self.doc_id = doc_id
+        doc_url = f'https://docs.google.com/spreadsheets/d/{doc_id}'
+        gsp = gspread.authorize(self.credentials)
+        doc = gsp.open_by_url(doc_url)
+        self.tab_index = tab_index
+        self.ws = doc.get_worksheet(tab_index)
+        self.header_info = None
+        self.header_info_reverse = None
+        self.unique_header = unique_header
+
+
+    def get_credentials(self, project_filepath=None):
+        if os.path.exists(self.credentials_filepath) == False:
+            logger.info(f"credentials_filepath : {self.credentials_filepath}")
+            url = self.__make_token_cli(project_filepath)
+            logger.debug(f"Auth URL : {url}")
+            code = input("Input Code : ")
+            self.__save_token(self.credentials_filepath, code)
+
+        store = Storage(self.credentials_filepath)
+        credentials = store.get()
+        if not credentials or credentials.invalid:
+            logger.warning('credentials error')
+            #flow = client.flow_from_clientsecrets('credentials.json', SCOPES)
+            #creds = tools.run_flow(flow, store)
+            os.remove(self.credentials_filepath)
+            return self.get_credentials(self.credentials_filepath)
+        return credentials
+    
+    
+    def __make_token_cli(self, project_filepath):
+        try:
+            if project_filepath == None:
+                project_filepath = os.path.join(os.path.dirname(__file__), 'cs.json')
+            self.current_flow  = flow_from_clientsecrets(
+                project_filepath,  # downloaded file
+                'https://www.googleapis.com/auth/drive',  # scope
+                redirect_uri='urn:ietf:wg:oauth:2.0:oob')
+            return self.current_flow.step1_get_authorize_url()
+        except Exception as e:
+            logger.error(f"Exception: {e}")
+            logger.error(traceback.format_exc())
+
+
+    def __save_token(self, credentials_filepath, code):
+        try:
+            credentials = self.current_flow.step2_exchange(code)
+            storage = Storage(credentials_filepath)
+            storage.put(credentials)
+            return True
+        except Exception as e:
+            logger.error(f"Exception: {e}")
+            logger.error(traceback.format_exc())
+            return False
+    
+
+    def get_sheet_data(self):
+        tmp = self.ws.get_all_values()#[:-1]
+        self.set_sheet_header(tmp[0])
+        rows = tmp[1:]
+        ret = []
+        for row in rows:
+            item = {}
+            for idx, col in enumerate(row):
+                item[self.header_info_reverse[idx+1]] = col
+            ret.append(item)
+        return ret
+
+
+    def set_sheet_header(self, row):
+        self.header_info = {}
+        self.header_info_reverse = {}
+        for idx, col in enumerate(row):
+            self.header_info[col] = idx + 1
+            self.header_info_reverse[idx+1] = col
+        logger.debug(self.header_info)
+
+    
+    def find_row_index(self, total_data, data):
+        find_row_index = -1
+        #find = False
+        #data['IDX'] = len(total_data)+1
+        for idx, item in enumerate(total_data):
+            if item[self.unique_header] == str(data[self.unique_header]):
+                #find = True
+                find_row_index = idx
+                #data['IDX'] = find_row_index + 1
+                break
+        
+        if find_row_index == -1:
+            data['IDX'] = len(total_data)+1
+
+        return find_row_index
+    
+
+    def sleep(self):
+        time.sleep(0.5)
+
+    def sleep_exception(self):
+        time.sleep(10)
+
+    
+    def after_update_cell(self, sheet_row_index, sheet_col_index, key, value, old_value):
+        pass
+    
+
+    def set_color(self, sheet_row, sheet_col1, sheet_col2, color):
+        format_cell_range(self.ws, gspread.utils.rowcol_to_a1(sheet_row,sheet_col1)+':' + gspread.utils.rowcol_to_a1(sheet_row,sheet_col2), color)
+    
+    def set_color_row(self, sheet_row, color):
+        format_cell_range(self.ws, gspread.utils.rowcol_to_a1(sheet_row,1)+':' + gspread.utils.rowcol_to_a1(sheet_row,len(self.header_info)), color)
+
+    def set_color_cell(self, sheet_row, sheet_col, color):
+        format_cell_range(self.ws, gspread.utils.rowcol_to_a1(sheet_row,sheet_col)+':' + gspread.utils.rowcol_to_a1(sheet_row,sheet_col), color)
+
+    def write_data(self, total_data, data):
+        find_row_index = self.find_row_index(total_data, data)
+
+        write_count = 0
+        for key, value in data.items():
+            if key.startswith('_'):
+                continue
+            if value == None:
+                continue
+            if key not in self.header_info:
+                continue
+            while True:
+                try:
+                    if find_row_index != -1 and str(total_data[find_row_index][key]) != str(value):
+                        logger.warning(f"업데이트 : {key} {total_data[find_row_index][key]} ==> {value}")
+                        self.ws.update_cell(find_row_index+2, self.header_info[key], value)
+                        self.after_update_cell(find_row_index+2, self.header_info[key], key, value, total_data[find_row_index][key])
+                        write_count += 1
+                        self.sleep()
+                    elif find_row_index == -1 and value != '':
+                        logger.warning(f"추가 : {key} {value}")
+                        self.ws.update_cell(len(total_data)+2, self.header_info[key], value)
+                        self.after_update_cell(len(total_data)+2, self.header_info[key], key, value, None)
+                        write_count += 1
+                        self.sleep()
+                    break
+                except gspread.exceptions.APIError:
+                    self.sleep_exception()
+                except Exception as exception: 
+                    logger.error(f"{key} - {value}")
+                    logger.error('Exception:%s', exception)
+                    logger.error(traceback.format_exc())
+                    logger.error(self.header_info)
+                    self.sleep_exception()
+          
+        if find_row_index == -1:
+            total_data.append(data)
+        else:
+            total_data[find_row_index] = data
+        return write_count