test

lib/tool_base/__init__.py

@@ -0,0 +1,20 @@
+from framework import logger
+from .notify import ToolBaseNotify
+from .file import ToolBaseFile
+from .aes_cipher import ToolAESCipher
+from .celery_shutil import ToolShutil
+from .subprocess import ToolSubprocess
+from .rclone import ToolRclone
+from .ffmpeg import ToolFfmpeg
+from .util import ToolUtil
+from .hangul import ToolHangul
+from .os_command import ToolOSCommand
+from .util_yaml import ToolUtilYaml
+
+
+def d(data):
+    if type(data) in [type({}), type([])]:
+        import json
+        return '\n' + json.dumps(data, indent=4, ensure_ascii=False)
+    else:
+        return str(data)

lib/tool_base/aes_cipher.py

@@ -0,0 +1,40 @@
+import base64
+from Crypto.Cipher import AES
+from Crypto import Random
+from framework import app, logger
+
+BS = 16
+pad = lambda s: s + (BS - len(s) % BS) * chr(BS - len(s) % BS) 
+unpad = lambda s : s[0:-s[-1]]
+key = b'140b41b22a29beb4061bda66b6747e14'
+
+class ToolAESCipher(object):
+    @staticmethod
+    def encrypt(raw, mykey=None):
+        try:
+            Random.atfork()
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc()) 
+
+        raw = pad(raw)
+        if type(raw) == type(''):
+            raw = raw.encode()
+        if mykey is not None and type(mykey) == type(''):
+            mykey = mykey.encode()
+        iv = Random.new().read( AES.block_size )
+        cipher = AES.new(key if mykey is None else mykey, AES.MODE_CBC, iv )
+        try:
+            tmp = cipher.encrypt( raw )
+        except:
+            tmp = cipher.encrypt( raw.encode() )
+        ret = base64.b64encode( iv + tmp ) 
+        ret = ret.decode()
+        return ret
+
+    @staticmethod
+    def decrypt(enc, mykey=None):
+        enc = base64.b64decode(enc)
+        iv = enc[:16]
+        cipher = AES.new(key if mykey is None else mykey, AES.MODE_CBC, iv )
+        return unpad(cipher.decrypt( enc[16:] ))

lib/tool_base/celery_shutil.py

@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+import os
+import traceback
+import shutil
+
+from framework import app, celery, logger
+
+class ToolShutil(object):
+    # run_in_celery=True 이미 celery안에서 실행된다. 바로 콜한다.
+    @staticmethod
+    def move(source_path, target_path, run_in_celery=False):
+        try:
+            if app.config['config']['use_celery'] and run_in_celery == False:
+                result = ToolShutil._move_task.apply_async((source_path, target_path))
+                return result.get()
+            else:
+                return ToolShutil._move_task(source_path, target_path)
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return ToolShutil._move_task(source_path, target_path)
+
+    @staticmethod
+    @celery.task
+    def _move_task(source_path, target_path):
+        try:
+            logger.debug('_move_task:%s %s', source_path, target_path)
+            shutil.move(source_path, target_path)
+            logger.debug('_move_task end')
+            return True
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return False
+
+    @staticmethod
+    def move_exist_remove(source_path, target_path, run_in_celery=False):
+        try:
+            if app.config['config']['use_celery'] and run_in_celery == False:
+                result = ToolShutil._move_exist_remove_task.apply_async((source_path, target_path))
+                return result.get()
+            else:
+                return ToolShutil._move_exist_remove_task(source_path, target_path)
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return ToolShutil._move_exist_remove_task(source_path, target_path)
+
+    @staticmethod
+    @celery.task
+    def _move_exist_remove_task(source_path, target_path):
+        try:
+            target_file_path = os.path.join(target_path, os.path.basename(source_path))
+            if os.path.exists(target_file_path):
+                os.remove(source_path)
+                return True
+            logger.debug('_move_exist_remove:%s %s', source_path, target_path)
+            shutil.move(source_path, target_path)
+            logger.debug('_move_exist_remove end')
+            return True
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return False
+
+    @staticmethod
+    def copytree(source_path, target_path):
+        try:
+            if app.config['config']['use_celery']:
+                result = ToolShutil._copytree_task.apply_async((source_path, target_path))
+                return result.get()
+            else:
+                return ToolShutil._copytree_task(source_path, target_path)
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return ToolShutil._copytree_task(source_path, target_path)
+
+    @staticmethod
+    @celery.task
+    def _copytree_task(source_path, target_path):
+        try:
+            shutil.copytree(source_path, target_path)
+            return True
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return False
+
+    # copy
+    @staticmethod
+    def copy(source_path, target_path):
+        try:
+            if app.config['config']['use_celery']:
+                result = ToolShutil._copy_task.apply_async((source_path, target_path))
+                return result.get()
+            else:
+                return ToolShutil._copy_task(source_path, target_path)
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return ToolShutil._copy_task(source_path, target_path)
+
+    @staticmethod
+    @celery.task
+    def _copy_task(source_path, target_path):
+        try:
+            shutil.copy(source_path, target_path)
+            return True
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return False
+
+    # rmtree
+    @staticmethod
+    def rmtree(source_path):
+        try:
+            if app.config['config']['use_celery']:
+                result = ToolShutil._rmtree_task.apply_async((source_path,))
+                return result.get()
+            else:
+                return ToolShutil._rmtree_task(source_path)
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return ToolShutil._rmtree_task(source_path)
+
+    @staticmethod
+    @celery.task
+    def _rmtree_task(source_path):
+        try:
+            shutil.rmtree(source_path)
+            return True
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return False   
+
+    @staticmethod
+    def remove(remove_path):
+        try:
+            logger.debug('CELERY os.remove start : %s', remove_path)
+            if app.config['config']['use_celery']:
+                result = ToolShutil._remove_task.apply_async((remove_path,))
+                return result.get()
+            else:
+                return ToolShutil._remove_task(remove_path)
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return ToolShutil._remove_task(remove_path)
+        finally:
+            logger.debug('CELERY os.remove end : %s', remove_path)
+
+    @staticmethod
+    @celery.task
+    def _remove_task(remove_path):
+        try:
+            os.remove(remove_path)
+            return True
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return False   

lib/tool_base/ffmpeg.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#########################################################
+import os, sys, traceback, subprocess, json, platform
+from framework import app, logger, path_data
+from .subprocess import ToolSubprocess
+
+class ToolFfmpeg(object):
+
+    @classmethod
+    def ffprobe(cls, filepath, ffprobe_path='ffprobe', option=None):
+        try:
+            command = [ffprobe_path, '-v', 'quiet', '-print_format', 'json', '-show_format', '-show_streams', filepath]
+            if option is not None:
+                command += option
+            logger.warning(' '.join(command))
+            ret = ToolSubprocess.execute_command_return(command, format='json')
+            return ret
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+

lib/tool_base/file.py

@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+#########################################################
+import os, traceback, io, re, json, codecs
+from . import logger
+
+class ToolBaseFile(object):
+
+    @classmethod
+    def read(cls, filepath, mode='r'):
+        try:
+            import codecs
+            ifp = codecs.open(filepath, mode, encoding='utf8')
+            data = ifp.read()
+            ifp.close()
+            if isinstance(data, bytes):
+                data = data.decode('utf-8') 
+            return data
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            
+
+    @classmethod
+    def download(cls, url, filepath):
+        try:
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
+                'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+                'Accept-Language' : 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
+                'Connection': 'Keep-Alive',
+            }
+
+            import requests
+            with open(filepath, "wb") as file_is:   # open in binary mode
+                response = requests.get(url, headers=headers)               # get request
+                file_is.write(response.content)      # write to file
+                return True
+        except Exception as exception:
+            logger.debug('Exception:%s', exception)
+            logger.debug(traceback.format_exc())   
+        return False
+
+
+    @classmethod
+    def write(cls, data, filepath, mode='w'):
+        try:
+            import codecs
+            ofp = codecs.open(filepath, mode, encoding='utf8')
+            if isinstance(data, bytes) and mode == 'w':
+                data = data.decode('utf-8') 
+            ofp.write(data)
+            ofp.close()
+            return True
+        except Exception as exception:
+            logger.debug('Exception:%s', exception)
+            logger.debug(traceback.format_exc())
+        return False
+
+
+    @classmethod
+    def text_for_filename(cls, text):
+        #text = text.replace('/', '')
+        # 2021-07-31 X:X
+        #text = text.replace(':', ' ')
+        text = re.sub('[\\/:*?\"<>|]', ' ', text).strip()
+        text = re.sub("\s{2,}", ' ', text)
+        return text
+
+
+    @classmethod
+    def size(cls, start_path = '.'):
+        total_size = 0
+        for dirpath, dirnames, filenames in os.walk(start_path):
+            for f in filenames:
+                fp = os.path.join(dirpath, f)
+                if not os.path.islink(fp):
+                    total_size += os.path.getsize(fp)
+        return total_size
+
+    @classmethod
+    def file_move(cls, source_path, target_dir, target_filename):
+        try:
+            import time, shutil
+            if os.path.exists(target_dir) == False:
+                os.makedirs(target_dir)
+            target_path = os.path.join(target_dir, target_filename)
+            if source_path != target_path:
+                if os.path.exists(target_path):
+                    tmp = os.path.splitext(target_filename)
+                    new_target_filename = f"{tmp[0]} {str(time.time()).split('.')[0]}{tmp[1]}"
+                    target_path = os.path.join(target_dir, new_target_filename)
+                shutil.move(source_path, target_path)
+        except Exception as exception:
+            logger.debug('Exception:%s', exception)
+            logger.debug(traceback.format_exc())
+
+
+    @classmethod
+    def makezip(cls, zip_path, zip_folder=None, zip_extension='zip', remove_folder=False):
+        import zipfile
+        try:
+            zip_path = zip_path.rstrip('/')
+            if zip_folder is None:
+                zip_folder = os.path.dirname(zip_path)
+            elif zip_folder == 'tmp':
+                from framework import path_data
+                zip_folder = os.path.join(path_data, 'tmp')
+            if os.path.isdir(zip_path):
+                zipfilepath = os.path.join(zip_folder, f"{os.path.basename(zip_path)}.{zip_extension}")
+                fantasy_zip = zipfile.ZipFile(zipfilepath, 'w')
+                for f in os.listdir(zip_path):
+                    src = os.path.join(zip_path, f)
+                    fantasy_zip.write(src, os.path.basename(src), compress_type = zipfile.ZIP_DEFLATED)
+                fantasy_zip.close()
+            if remove_folder:
+                import shutil
+                shutil.rmtree(zip_path)
+            return zipfilepath
+        except Exception as exception:
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+        return
+
+    @classmethod
+    def rmtree(cls, folderpath):
+        import shutil
+        try:
+            shutil.rmtree(folderpath)
+            return True
+        except:
+            try:
+                os.system("rm -rf '{folderpath}'")
+                return True
+            except:
+                return False
+            
+                
+    @classmethod
+    def rmtree2(cls, folderpath):
+        import shutil
+        try:
+            for root, dirs, files in os.walk(folderpath):
+                for name in files:
+                    os.remove(os.path.join(root, name))
+                for name in dirs:
+                    shutil.rmtree(os.path.join(root, name))
+        except:
+            return False
+          
+    
+    @classmethod
+    def write_json(cls, filepath, data):
+        try:
+            if os.path.exists(os.path.dirname(filepath)) == False:
+                os.makedirs(os.path.dirname(filepath))
+            with open(filepath, "w", encoding='utf8') as json_file:
+                json.dump(data, json_file, indent=4, ensure_ascii=False)
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc()) 
+
+    @classmethod
+    def read_json(cls, filepath):
+        try:
+            with open(filepath, "r", encoding='utf8') as json_file:
+                data = json.load(json_file)
+                return data
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc()) 
+    
+
+    @classmethod
+    def write_file(cls, filename, data):
+        try:
+            import codecs
+            ofp = codecs.open(filename, 'w', encoding='utf8')
+            ofp.write(data)
+            ofp.close()
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc()) 
+
+    @classmethod
+    def read_file(cls, filename):
+        try:
+            ifp = codecs.open(filename, 'r', encoding='utf8')
+            data = ifp.read()
+            ifp.close()
+            return data
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+
+    
+
+    @classmethod
+    def makezip_simple(cls, zip_path, zip_extension='cbz', remove_zip_path=True):
+        import zipfile, shutil
+        try:
+            if os.path.exists(zip_path) == False:
+                return False
+            zipfilepath = os.path.join(os.path.dirname(zip_path), f"{os.path.basename(zip_path)}.{zip_extension}")
+            if os.path.exists(zipfilepath):
+                return True
+            zip = zipfile.ZipFile(zipfilepath, 'w')
+            for f in os.listdir(zip_path):
+                src = os.path.join(zip_path, f)
+                zip.write(src, f, compress_type = zipfile.ZIP_DEFLATED)
+            zip.close()
+            if remove_zip_path:
+                shutil.rmtree(zip_path)
+            return zipfilepath
+        except Exception as e:
+            logger.error(f'Exception:{str(e)}')
+            logger.error(traceback.format_exc())
+        return None
+        

lib/tool_base/hangul.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+#########################################################
+import os, traceback, io, re
+from . import logger
+
+class ToolHangul(object):
+
+
+    @classmethod
+    def is_include_hangul(cls, text):
+        try:
+            hanCount = len(re.findall(u'[\u3130-\u318F\uAC00-\uD7A3]+', text))
+            return hanCount > 0
+        except:
+            return False
+
+    @classmethod
+    def language_info(cls, text):
+        try:
+            text = text.strip().replace(' ', '')
+            all_count = len(text)
+            han_count = len(re.findall('[\u3130-\u318F\uAC00-\uD7A3]', text))
+            eng_count = len(re.findall('[a-zA-Z]', text))
+            han_percent = int(han_count * 100 / all_count)
+            eng_percent = int(eng_count * 100 / all_count)
+            return (han_percent, eng_percent)
+        except:
+            return False

lib/tool_base/notify.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+#########################################################
+
+import os
+import traceback
+
+from discord_webhook import DiscordWebhook, DiscordEmbed
+from telepot2 import Bot, glance
+from telepot2.loop import MessageLoop
+
+from . import logger
+
+class ToolBaseNotify(object):
+
+    @classmethod
+    def send_message(cls, text, message_id=None, image_url=None):
+        from system.model import ModelSetting as SystemModelSetting
+        if SystemModelSetting.get_bool('notify_advaned_use'):
+            return cls.send_advanced_message(text, image_url=image_url, message_id=message_id)
+        else:
+            if SystemModelSetting.get_bool('notify_telegram_use'):
+                cls.send_telegram_message(text, image_url=image_url, bot_token=SystemModelSetting.get('notify_telegram_token'), chat_id=SystemModelSetting.get('notify_telegram_chat_id'))
+            if SystemModelSetting.get_bool('notify_discord_use'):
+                cls.send_discord_message(text, image_url=image_url, webhook_url=SystemModelSetting.get('notify_discord_webhook')) 
+
+    @classmethod
+    def send_advanced_message(cls, text, image_url=None, policy=None, message_id=None):
+        from system.model import ModelSetting as SystemModelSetting
+        try:
+            if policy is None:
+                policy = SystemModelSetting.get('notify_advaned_policy')
+
+            if message_id is None:
+                message_id = 'DEFAULT'
+            
+            policy_list = cls._make_policy_dict(policy)
+            #logger.debug(policy_list)
+            #logger.debug(message_id)
+            if message_id.strip() not in policy_list:
+                message_id = 'DEFAULT'
+            
+            for tmp in policy_list[message_id.strip()]:
+                if tmp.startswith('http'):
+                    cls.send_discord_message(text, image_url=image_url, webhook_url=tmp)
+                elif tmp.find(',') != -1:
+                    tmp2 = tmp.split(',')
+                    cls.send_telegram_message(text, image_url=image_url, bot_token=tmp2[0], chat_id=tmp2[1])
+            return True
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc()) 
+            #logger.debug('Chatid:%s', chat_id)
+        return False
+
+    @classmethod
+    def _make_policy_dict(cls, policy):
+        try:
+            ret = {}
+            for t in policy.split('\n'):
+                t = t.strip()
+                if t == '' or t.startswith('#'):
+                    continue
+                else:
+                    tmp2 = t.split('=')
+                    if len(tmp2) != 2:
+                        continue
+                    ret[tmp2[0].strip()] = [x.strip() for x in tmp2[1].split('|')]
+            return ret
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc()) 
+        return False
+
+    @classmethod
+    def send_discord_message(cls, text, image_url=None, webhook_url=None):
+        from system.model import ModelSetting as SystemModelSetting
+        try:
+            if webhook_url is None:
+                webhook_url = SystemModelSetting.get('notify_discord_webhook')
+            
+            webhook = DiscordWebhook(url=webhook_url, content=text)
+            if image_url is not None:
+                embed = DiscordEmbed()
+                embed.set_timestamp()
+                embed.set_image(url=image_url)
+                webhook.add_embed(embed)
+            response = webhook.execute()
+            #discord = response.json()
+            #logger.debug(discord)
+            return True
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc()) 
+        return False
+
+    @classmethod
+    def send_telegram_message(cls, text, bot_token=None, chat_id=None, image_url=None,  disable_notification=None):
+        from system.model import ModelSetting as SystemModelSetting
+        try:
+            if bot_token is None:
+                bot_token = SystemModelSetting.get('notify_telegram_token')
+            
+            if chat_id is None:
+                chat_id = SystemModelSetting.get('notify_telegram_chat_id')
+
+            if disable_notification is None:
+                disable_notification = SystemModelSetting.get_bool('notify_telegram_disable_notification')
+
+            bot = Bot(bot_token)
+            if image_url is not None:
+                #bot.sendPhoto(chat_id, text, caption=caption, disable_notification=disable_notification)
+                bot.sendPhoto(chat_id, image_url, disable_notification=disable_notification)
+            bot.sendMessage(chat_id, text, disable_web_page_preview=True, disable_notification=disable_notification)
+            #elif mime == 'video':
+            #    bot.sendVideo(chat_id, text, disable_notification=disable_notification)
+            return True
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc()) 
+            logger.debug('Chatid:%s', chat_id)
+        return False
+

lib/tool_base/os_command.py

@@ -0,0 +1,42 @@
+import os, traceback, io, re, platform
+from . import logger
+from . import ToolSubprocess
+from . import ToolUtil, ToolBaseFile
+
+class ToolOSCommand(object):
+
+    @classmethod
+    def get_size(cls, path):
+        if platform.system() == 'Windows':
+            #https://docs.microsoft.com/en-us/sysinternals/downloads/du
+            """
+            bin = r'C:\SJVA3\data\bin\du64.exe'
+            command = [bin, '-c', '-nobanner', f'"{path}"']
+            data = ToolSubprocess.execute_command_return(command, force_log=True)
+            logger.warning(data)
+            ret = {}
+            tmp = data.split('\t')
+            ret['target'] = tmp[1].strip()
+            ret['size'] = int(tmp[0].strip())
+            ret['sizeh'] = ToolUtil.sizeof_fmt(ret['size'])
+            """
+            ret = {}
+            ret['target'] = path
+            if os.path.exists(path):
+                if os.path.isdir(path):
+                    ret['size'] = ToolBaseFile.size(start_path=path)
+                else:
+                    ret['size'] = os.stat(path).st_size
+            ret['sizeh'] = ToolUtil.sizeof_fmt(ret['size'])
+            return ret
+
+        else:
+            command = ['du', '-bs', path]
+            data = ToolSubprocess.execute_command_return(command)
+            ret = {}
+            tmp = data.split('\t')
+            ret['target'] = tmp[1].strip()
+            ret['size'] = int(tmp[0].strip())
+            ret['sizeh'] = ToolUtil.sizeof_fmt(ret['size'])
+        return ret
+            

lib/tool_base/rclone.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+#########################################################
+import os, sys, traceback, subprocess, json, platform
+from framework import app, logger, path_data
+from .subprocess import ToolSubprocess
+
+class ToolRclone(object):
+
+    @classmethod
+    def lsjson(cls, remote_path, rclone_path='rclone', config_path=os.path.join(path_data, 'db', 'rclone.conf'), option=None):
+        try:
+            command = [rclone_path, '--config', config_path, 'lsjson', remote_path]
+            if option is not None:
+                command += option
+            logger.warning(' '.join(command))
+            ret = ToolSubprocess.execute_command_return(command, format='json')
+            if ret is not None:
+                ret = list(sorted(ret, key=lambda k:k['Path']))
+            return ret
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+
+
+    @classmethod
+    def size(cls, remote_path, rclone_path='rclone', config_path=os.path.join(path_data, 'db', 'rclone.conf'), option=None):
+        try:
+            command = [rclone_path, '--config', config_path, 'size', remote_path, '--json']
+            if option is not None:
+                command += option
+            ret = ToolSubprocess.execute_command_return(command, format='json')
+            return ret
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+    
+
+    @classmethod
+    def getid(cls, remote_path, rclone_path='rclone', config_path=os.path.join(path_data, 'db', 'rclone.conf'), option=None):
+        try:
+            command = [rclone_path, '--config', config_path, 'backend', 'getid', remote_path]
+            if option is not None:
+                command += option
+            ret = ToolSubprocess.execute_command_return(command)
+            if ret is not None and (len(ret.split(' ')) > 1 or ret == ''):
+                ret = None
+            return ret
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+    
+
+    @classmethod
+    def rmdir(cls, remote_path, rclone_path='rclone', config_path=os.path.join(path_data, 'db', 'rclone.conf'), option=None):
+        try:
+            command = [rclone_path, '--config', config_path, 'rmdir', remote_path, '-vv']#, '--drive-use-trash=false', '-vv']
+            if option is not None:
+                command += option
+            ret = ToolSubprocess.execute_command_return(command)
+            return ret
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+
+    @classmethod
+    def purge(cls, remote_path, rclone_path='rclone', config_path=os.path.join(path_data, 'db', 'rclone.conf'), option=None):
+        try:
+            command = [rclone_path, '--config', config_path, 'purge', remote_path, '-vv']#, '--drive-use-trash=false', '-vv']
+            if option is not None:
+                command += option
+            ret = ToolSubprocess.execute_command_return(command)
+            return ret
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+
+
+    @classmethod
+    def config_list(cls, rclone_path='rclone', config_path=os.path.join(path_data, 'db', 'rclone.conf'), option=None):
+        try:
+            command = [rclone_path, '--config', config_path, 'config', 'dump']#, '--drive-use-trash=false', '-vv']
+            if option is not None:
+                command += option
+            ret = ToolSubprocess.execute_command_return(command, format='json')
+            for key, value in ret.items():
+                if 'token' in value and value['token'].startswith('{'):
+                    value['token'] = json.loads(value['token'])
+            return ret
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+
+
+
+
+
+    @classmethod
+    def move_server_side(cls, source, target, rclone_path='rclone', config_path=os.path.join(path_data, 'db', 'rclone.conf'), option=None):
+        try:
+            
+            command = [rclone_path, '--config', config_path, 'move', source, target, '--drive-server-side-across-configs=true', '--delete-empty-src-dirs', '-vv']
+            if option is not None:
+                command += option
+            logger.debug(' '.join(command))
+            ret = ToolSubprocess.execute_command_return(command)
+            return ret
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+    
+
+    
+    @classmethod
+    def get_config(cls, remote_name, rclone_path='rclone', config_path=os.path.join(path_data, 'db', 'rclone.conf'), option=None):
+        try:
+            data = cls.config_list(rclone_path=rclone_path, config_path=config_path, option=option)
+            return data.get(remote_name, None)
+            
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+
+
+    @classmethod
+    def copy_server_side(cls, source, target, rclone_path='rclone', config_path=os.path.join(path_data, 'db', 'rclone.conf'), option=None):
+        try:
+            command = [rclone_path, '--config', config_path, 'copy', source, target, '--drive-server-side-across-configs=true', '--drive-stop-on-upload-limit', '-vv']
+            if option is not None:
+                command += option
+            logger.debug(' '.join(command))
+            ret = ToolSubprocess.execute_command_return(command)
+            return ret
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())

lib/tool_base/subprocess.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+#########################################################
+import os, traceback, subprocess, json
+from framework import frame
+
+
+class ToolSubprocess(object):
+
+    @classmethod
+    def execute_command_return(cls, command, format=None, force_log=False, shell=False, env=None):
+        try:
+            #logger.debug('execute_command_return : %s', ' '.join(command))
+            if frame.config['running_type'] == 'windows':
+                tmp = []
+                if type(command) == type([]):
+                    for x in command:
+                        if x.find(' ') == -1:
+                            tmp.append(x)
+                        else:
+                            tmp.append(f'"{x}"')
+                    command = ' '.join(tmp)
+
+            iter_arg =  ''
+            process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, shell=shell, env=env, encoding='utf8')
+            
+            #process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, shell=shell, env=env, encoding='utf8')
+            ret = []
+            with process.stdout:
+                for line in iter(process.stdout.readline, iter_arg):
+                    ret.append(line.strip())
+                    if force_log:
+                        logger.debug(ret[-1])
+                process.wait() # wait for the subprocess to exit
+
+            if format is None:
+                ret2 = '\n'.join(ret)
+            elif format == 'json':
+                try:
+                    index = 0
+                    for idx, tmp in enumerate(ret):
+                        #logger.debug(tmp)
+                        if tmp.startswith('{') or tmp.startswith('['):
+                            index = idx
+                            break
+                    ret2 = json.loads(''.join(ret[index:]))
+                except:
+                    ret2 = None
+
+            return ret2
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            logger.error('command : %s', command)
+
+
+    # 2021-10-25
+    # timeout 적용
+    @classmethod
+    def execute_command_return2(cls, command, format=None, force_log=False, shell=False, env=None, timeout=None, uid=0, gid=0, pid_dict=None):
+        def demote(user_uid, user_gid):
+            def result():
+                os.setgid(user_gid)
+                os.setuid(user_uid)
+            return result
+        try:
+            if app.config['config']['running_type'] == 'windows':
+                tmp = []
+                if type(command) == type([]):
+                    for x in command:
+                        if x.find(' ') == -1:
+                            tmp.append(x)
+                        else:
+                            tmp.append(f'"{x}"')
+                    command = ' '.join(tmp)
+
+            iter_arg =  b'' if app.config['config']['is_py2'] else ''
+            if app.config['config']['running_type'] == 'windows':
+                process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, shell=shell, env=env, encoding='utf8')
+            else:
+                process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, shell=shell, env=env, preexec_fn=demote(uid, gid), encoding='utf8')
+                
+
+            new_ret = {'status':'finish', 'log':None}
+            try:
+                process_ret = process.wait(timeout=timeout) # wait for the subprocess to exit
+            except:
+                import psutil
+                process = psutil.Process(process.pid)
+                for proc in process.children(recursive=True):
+                    proc.kill()
+                process.kill()
+                new_ret['status'] = "timeout"
+
+            ret = []
+            with process.stdout:
+                for line in iter(process.stdout.readline, iter_arg):
+                    ret.append(line.strip())
+                    if force_log:
+                        logger.debug(ret[-1])
+
+            if format is None:
+                ret2 = '\n'.join(ret)
+            elif format == 'json':
+                try:
+                    index = 0
+                    for idx, tmp in enumerate(ret):
+                        #logger.debug(tmp)
+                        if tmp.startswith('{') or tmp.startswith('['):
+                            index = idx
+                            break
+                    ret2 = json.loads(''.join(ret[index:]))
+                except:
+                    ret2 = None
+
+            new_ret['log'] = ret2
+            return new_ret
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            logger.error('command : %s', command)

lib/tool_base/util.py

@@ -0,0 +1,56 @@
+import os, sys, traceback, json
+
+from . import logger
+
+class ToolUtil(object):
+
+    @classmethod
+    def make_apikey_url(cls, url):
+        from framework import SystemModelSetting
+        if not url.startswith('http'):
+            url = SystemModelSetting.get('ddns') + url
+        if SystemModelSetting.get_bool('auth_use_apikey'):
+            if url.find('?') == -1:
+                url += '?'
+            else:
+                url += '&'
+            url += 'apikey=%s' % SystemModelSetting.get('auth_apikey')
+        return url
+    
+    @classmethod
+    def save_dict(cls, data, filepath):
+        try:
+            import json, codecs
+            data = json.dumps(data, indent=4, ensure_ascii=False)
+            ofp = codecs.open(filepath, 'w', encoding='utf8')
+            ofp.write(data)
+            ofp.close()
+        except Exception as exception:
+            logger.debug('Exception:%s', exception)
+            logger.debug(traceback.format_exc())
+
+
+    @classmethod
+    def dump(cls, data):
+        if type(data) in [type({}), type([])]:
+            return '\n' + json.dumps(data, indent=4, ensure_ascii=False)
+        else:
+            return str(data)
+
+
+
+    @classmethod
+    def sizeof_fmt(cls, num, suffix='B'):
+        for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
+            if abs(num) < 1024.0:
+                return "%3.1f%s%s" % (num, unit, suffix)
+            num /= 1024.0
+        return "%.2f%s%s" % (num, 'Y', suffix)
+
+
+    @classmethod
+    def timestamp_to_datestr(cls, stamp, format='%Y-%m-%d %H:%M:%S'):
+        from datetime import datetime
+        tmp = datetime.fromtimestamp(stamp)
+        return tmp.strftime(format)
+    

lib/tool_base/util_yaml.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+#########################################################
+import os, sys, traceback, subprocess, json, platform
+from framework import app, logger, path_data
+from .file import ToolBaseFile
+
+class ToolUtilYaml(object):
+
+    @classmethod
+    def copy_section(cls, source_file, target_file, section_name):
+        try:
+            if os.path.exists(source_file) == False:
+                return 'not_exist_source_file'
+            if os.path.exists(target_file) == False:
+                return 'not_exist_target_file'
+            lines = ToolBaseFile.read(source_file).split('\n')
+            section = {}
+            current_section_name = None
+            current_section_data = None
+
+            for line in lines:
+                line = line.strip()
+                if line.startswith('# SECTION START : '):
+                    current_section_name = line.split(':')[1].strip()
+                    current_section_data = []
+                if current_section_data is not None:
+                    current_section_data.append(line)
+                if line.startswith('# SECTION END'):
+                    section[current_section_name] = current_section_data
+                    current_section_name = current_section_data = None
+
+            if section_name not in section:
+                return 'not_include_section'
+            
+            data = '\n'.join(section[section_name])
+            source_data = ToolBaseFile.read(target_file)
+            source_data = source_data + f"\n{data}\n"
+            ToolBaseFile.write(source_data, target_file)
+            return 'success'
+        except Exception as exception: 
+            logger.error('Exception:%s', exception)
+            logger.error(traceback.format_exc())
+            return 'exception'