From 751adbbedc5d4af8f3862bdff42eeca61b260307 Mon Sep 17 00:00:00 2001 From: flaskfarm Date: Wed, 12 Oct 2022 01:32:51 +0900 Subject: [PATCH] update --- .gitignore | 3 +- lib/framework/init_main.py | 6 +- lib/framework/init_route.py | 9 + lib/framework/static/js/ff_common1.js | 5 +- lib/framework/static/js/ff_global1.js | 70 +++++-- lib/framework/static/js/sjva_global1.js | 8 +- lib/framework/templates/macro_include.html | 8 +- lib/support/__init__.py | 30 +-- lib/support/base/__init__.py | 12 +- lib/support/base/subprocess.py | 175 +++++++++++++++++- lib/support/logger.py | 9 +- lib/system/mod_setting.py | 65 ++++--- lib/system/mod_tool.py | 2 +- lib/system/setup.py | 3 +- .../templates/system_setting_celery.html | 123 ++++++++++++ lib/system/templates/system_tool_crypt.html | 2 +- lib/tool/__init__.py | 1 + lib/tool/modal_command.py | 113 +++++++++++ 18 files changed, 548 insertions(+), 96 deletions(-) create mode 100644 lib/system/templates/system_setting_celery.html create mode 100644 lib/tool/modal_command.py diff --git a/.gitignore b/.gitignore index 8488add..86de131 100644 --- a/.gitignore +++ b/.gitignore @@ -140,4 +140,5 @@ export.sh run.sh pre_start.sh *.code-workspace -false/ \ No newline at end of file +false +*copy.py \ No newline at end of file diff --git a/lib/framework/init_main.py b/lib/framework/init_main.py index f56030e..45bbb7c 100644 --- a/lib/framework/init_main.py +++ b/lib/framework/init_main.py @@ -61,9 +61,7 @@ class Framework: self.__make_default_dir() self.logger = self.get_logger(__package__) - - from support import set_logger - set_logger(self.logger) + import support self.__prepare_starting() self.app = Flask(__name__) @@ -462,6 +460,8 @@ class Framework: def __app_close(self): try: + from support import SupportSubprocess + SupportSubprocess.all_process_close() from .init_plugin import PluginManager PluginManager.plugin_unload() self.socketio.stop() diff --git a/lib/framework/init_route.py b/lib/framework/init_route.py index b8ccbf0..a68853e 100644 --- a/lib/framework/init_route.py +++ b/lib/framework/init_route.py @@ -40,6 +40,15 @@ def global_ajax(sub): return jsonify(ret) except: return jsonify({'ret':False}) + elif sub == 'command_modal_hide': + from tool import ToolModalCommand + ToolModalCommand.modal_close() + return jsonify('') + elif sub == 'command_modal_input': + from tool import ToolModalCommand + cmd = request.form['cmd'] + ToolModalCommand.input_command(cmd) + return jsonify('') diff --git a/lib/framework/static/js/ff_common1.js b/lib/framework/static/js/ff_common1.js index 69759f9..e9a27bd 100644 --- a/lib/framework/static/js/ff_common1.js +++ b/lib/framework/static/js/ff_common1.js @@ -21,7 +21,10 @@ $(window).on("load resize", function (event) { $body.css("padding-top", $navbar.outerHeight()); }); - +$('#command_modal').on('show.bs.modal', function (event) { + console.log('111111111') + console.log(event); +}) /////////////////////////////////////// // 사용 미확인 diff --git a/lib/framework/static/js/ff_global1.js b/lib/framework/static/js/ff_global1.js index e4e866e..8aca164 100644 --- a/lib/framework/static/js/ff_global1.js +++ b/lib/framework/static/js/ff_global1.js @@ -29,19 +29,6 @@ frameSocket.on('modal', function(data){ m_modal(data.data, data.title, false); }); -frameSocket.on('command_modal_add_text', function(data){ - document.getElementById("command_modal_textarea").innerHTML += data ; - document.getElementById("command_modal_textarea").scrollTop = document.getElementById("command_modal_textarea").scrollHeight; -}); - -frameSocket.on('command_modal_show', function(data){ - command_modal_show(data) -}); - -frameSocket.on('command_modal_clear', function(data){ - document.getElementById("command_modal_textarea").innerHTML = "" -}); - frameSocket.on('loading_hide', function(data){ $('#loading').hide(); }); @@ -52,6 +39,16 @@ frameSocket.on('refresh', function(data){ }); +$('#command_modal').on('hide.bs.modal', function (event) { + $.ajax({ + url: `/global/ajax/command_modal_hide`, + type: 'POST', + cache: false, + data: {}, + dataType: 'json' + }); +}); + @@ -250,3 +247,50 @@ let listdir = (path = '/', only_dir = true) => { // 파일 선택 모달 End /////////////////////////////////////// + +/////////////////////////////////////// +// Command MODAL +/////////////////////////////////////// + +frameSocket.on('command_modal_add_text', function(data){ + document.getElementById("command_modal_textarea").innerHTML += data ; + document.getElementById("command_modal_textarea").scrollTop = document.getElementById("command_modal_textarea").scrollHeight; +}); + +frameSocket.on('command_modal_input_disable', function(data){ + $('#command_modal_input').attr('disabled', true); +}); + +frameSocket.on('command_modal_show', function(data){ + command_modal_show(data) +}); + +frameSocket.on('command_modal_clear', function(data){ + document.getElementById("command_modal_textarea").innerHTML = "" +}); + +function command_modal_show(title) { + ClientHeight = window.innerHeight + document.getElementById("command_modal_title").innerHTML = title + $("#command_modal").height(ClientHeight+50); + $("#command_modal_textarea").height(ClientHeight-380); + $("#command_modal").modal({backdrop: 'static', keyboard: false}, 'show'); + $('#command_modal_input').attr('disabled', false); +} + +$("body").on('click', '#command_modal_input_btn', function(e) { + e.preventDefault(); + $.ajax({ + url: '/global/ajax/command_modal_input', + type: "POST", + cache: false, + data: {cmd:$('#command_modal_input').val()}, + dataType: "json", + success: function (ret) { + $('#command_modal_input').val(''); + } + }); +}); + +/////////////////////////////////////// + diff --git a/lib/framework/static/js/sjva_global1.js b/lib/framework/static/js/sjva_global1.js index 15d752d..f576acd 100644 --- a/lib/framework/static/js/sjva_global1.js +++ b/lib/framework/static/js/sjva_global1.js @@ -66,13 +66,7 @@ $("body").on('click', '#global_downloader_add_btn', function(e){ }); }); -function command_modal_show(title) { - ClientHeight = window.innerHeight - document.getElementById("command_modal_title").innerHTML = title - $("#command_modal").height(ClientHeight-100); - $("#command_modal_textarea").height(ClientHeight-380); - $("#command_modal").modal(); -} + // 토렌트 프로그램에 다운로드 추가할 결과를 보여주는 diff --git a/lib/framework/templates/macro_include.html b/lib/framework/templates/macro_include.html index 90a5abf..95daeef 100644 --- a/lib/framework/templates/macro_include.html +++ b/lib/framework/templates/macro_include.html @@ -116,8 +116,14 @@ {{ macros.m_modal_start('command_modal', '', 'modal-lg') }}
- +
+
+
+ + +
+
{{ macros.m_modal_end() }} diff --git a/lib/support/__init__.py b/lib/support/__init__.py index 8d0b56e..bea18d8 100644 --- a/lib/support/__init__.py +++ b/lib/support/__init__.py @@ -8,31 +8,9 @@ def d(data): else: return str(data) -def load(): - from .base.aes import SupportAES - from .base.discord import SupportDiscord - from .base.file import SupportFile - from .base.process import SupportProcess - from .base.string import SupportString - from .base.subprocess import SupportSubprocess - from .base.telegram import SupportTelegram - from .base.util import (AlchemyEncoder, SingletonClass, SupportUtil, - default_headers, pt) - from .base.yaml import SupportYaml - -import os - -logger = None - -if os.environ.get('FF') == 'true': - def set_logger(l): - global logger - logger = l - -else: - from .logger import get_logger - logger = get_logger() +from .logger import get_logger +logger = get_logger() from .base.aes import SupportAES from .base.discord import SupportDiscord @@ -44,7 +22,3 @@ from .base.telegram import SupportTelegram from .base.util import (AlchemyEncoder, SingletonClass, SupportUtil, default_headers, pt) from .base.yaml import SupportYaml - -# 일반 cli 사용 겸용이다. -# set_logger 로 인한 진입이 아니고 import가 되면 기본 경로로 로그파일을 -# 생성하기 때문에, set_logger 전에 import가 되지 않도록 주의. diff --git a/lib/support/base/__init__.py b/lib/support/base/__init__.py index 6974f4f..dda32cd 100644 --- a/lib/support/base/__init__.py +++ b/lib/support/base/__init__.py @@ -1,12 +1,4 @@ from support import d, logger -from .aes import SupportAES -from .discord import SupportDiscord -from .ffmpeg import SupportFfmpeg -from .file import SupportFile -from .image import SupportImage -from .process import SupportProcess -from .string import SupportString -from .util import (AlchemyEncoder, SingletonClass, SupportUtil, - default_headers, pt) -from .yaml import SupportYaml +import support +logger = support.logger diff --git a/lib/support/base/subprocess.py b/lib/support/base/subprocess.py index 5cacf40..af40edb 100644 --- a/lib/support/base/subprocess.py +++ b/lib/support/base/subprocess.py @@ -1,23 +1,29 @@ +import io import json import os import platform +import queue import subprocess +import threading +import time import traceback from . import logger +def demote(user_uid, user_gid): + def result(): + os.setgid(user_gid) + os.setuid(user_uid) + return result + class SupportSubprocess(object): # 2021-10-25 # timeout 적용 @classmethod def execute_command_return(cls, command, format=None, force_log=False, shell=False, env=None, timeout=None, uid=0, gid=0): - def demote(user_uid, user_gid): - def result(): - os.setgid(user_gid) - os.setuid(user_uid) - return result + try: if platform.system() == 'Windows': tmp = [] @@ -74,3 +80,162 @@ class SupportSubprocess(object): logger.error('Exception:%s', exception) logger.error(traceback.format_exc()) logger.error('command : %s', command) + + + instance_list = [] + + + def __init__(self, command, print_log=False, shell=False, env=None, timeout=None, uid=0, gid=0, stdout_callback=None): + self.command = command + self.print_log = print_log + self.shell = shell + self.env = env + self.timeout = timeout + self.uid = uid + self.gid = gid + self.stdout_callback = stdout_callback + self.process = None + self.stdout_queue = None + + + def start(self, join=True): + try: + self.thread = threading.Thread(target=self.execute_thread_function, args=()) + self.thread.setDaemon(True) + self.thread.start() + if join: + self.thread.join() + except Exception as e: + logger.error(f'Exception:{str(e)}') + logger.error(traceback.format_exc()) + + + def execute_thread_function(self): + try: + if platform.system() == 'Windows': + tmp = [] + if type(self.command) == type([]): + for x in self.command: + if x.find(' ') == -1: + tmp.append(x) + else: + tmp.append(f'"{x}"') + self.command = ' '.join(tmp) + + iter_arg = '' + if platform.system() == 'Windows': + self.process = subprocess.Popen(self.command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, shell=self.shell, env=self.env, encoding='utf8') + else: + self.process = subprocess.Popen(self.command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, shell=self.shell, env=self.env, preexec_fn=demote(self.uid, self.gid), encoding='utf8') + SupportSubprocess.instance_list.append(self) + self.start_communicate() + self.start_send_callback() + if self.process is not None: + self.process.wait() + #logger.info(f"{self.command} 정상 종료") + if self.stdout_queue != None: + self.stdout_queue.put('') + except Exception as e: + logger.error(f'Exception:{str(e)}') + logger.error(traceback.format_exc()) + + + + def start_communicate(self): + self.stdout_queue = queue.Queue() + sout = io.open(self.process.stdout.fileno(), 'rb', closefd=False) + + def Pump(stream): + _queue = queue.Queue() + + def rdr(): + while True: + if self.process == None: + break + buf = self.process.stdout.read(1) + if buf: + _queue.put( buf ) + else: + _queue.put( None ) + break + _queue.put( None ) + time.sleep(1) + + def clct(): + active = True + while active: + r = _queue.get() + if r is None: + break + try: + while True: + r1 = _queue.get(timeout=0.005) + if r1 is None: + active = False + break + else: + r += r1 + except: + pass + if r is not None: + if self.stdout_queue != None: + self.stdout_queue.put(r) + if self.stdout_queue != None: # 사용자 중지 + self.stdout_queue.put('') + for tgt in [rdr, clct]: + th = threading.Thread(target=tgt) + th.setDaemon(True) + th.start() + Pump(sout) + + + def start_send_callback(self): + def func(): + while self.stdout_queue: + line = self.stdout_queue.get() + if line == '': + if self.stdout_callback != None: + self.stdout_callback('end', None) + break + else: + if self.stdout_callback != None: + self.stdout_callback('log', line) + self.send_to_ui_thread = None + self.stdout_queue = None + self.process = None + + th = threading.Thread(target=func, args=()) + th.setDaemon(True) + th.start() + + + def process_close(self): + try: + if self.process is not None and self.process.poll() is None: + #import psutil + #process = psutil.Process(instance.process.pid) + #for proc in instance.process.children(recursive=True): + # proc.kill() + self.process.kill() + except Exception as e: + logger.error(f'Exception:{str(e)}') + logger.error(traceback.format_exc()) + finally: + try: + self.stdout_queue = None + self.process.kill() + except: pass + + def input_command(self, cmd): + if self.process != None: + self.process.stdin.write(f'{cmd}\n') + self.process.stdin.flush() + + + @classmethod + def all_process_close(cls): + for instance in cls.instance_list: + instance.process_close() + cls.instance_list = [] + + \ No newline at end of file diff --git a/lib/support/logger.py b/lib/support/logger.py index df485a2..4dd8708 100644 --- a/lib/support/logger.py +++ b/lib/support/logger.py @@ -1,6 +1,11 @@ -import os, sys, logging, logging.handlers +import logging +import logging.handlers +import os +import sys from datetime import datetime + from pytz import timezone, utc + """ ConsoleColor.Black => "\x1B[30m", ConsoleColor.DarkRed => "\x1B[31m", @@ -48,6 +53,8 @@ class CustomFormatter(logging.Formatter): def get_logger(name=None, log_path=None): + if os.environ.get('FF') == 'true': + name = 'framework' if name == None: name = sys.argv[0].rsplit('.', 1)[0] logger = logging.getLogger(name) diff --git a/lib/system/mod_setting.py b/lib/system/mod_setting.py index a1c05bf..979e240 100644 --- a/lib/system/mod_setting.py +++ b/lib/system/mod_setting.py @@ -2,6 +2,7 @@ import random import string from support import SupportDiscord, SupportFile, SupportTelegram +from tool.modal_command import ToolModalCommand from .setup import * @@ -33,6 +34,10 @@ class ModuleSetting(PluginModuleBase): 'notify_advaned_use' : 'False', 'notify.yaml': '', #직접 사용하지 않으나 저장 편의상. 'command_text': '', + 'celery_start_by_web': 'False', #웹 실행시 celery 실행 + 'celery_start_command': "celery -A flaskfarm.main.celery worker --loglevel=info --pool=gevent --concurrency=2 --config_filepath={F.config['config_filepath']} --running_type=native", + + } def __init__(self, P): @@ -57,6 +62,9 @@ class ModuleSetting(PluginModuleBase): elif page == 'notify': arg['notify_yaml_filepath'] = F.config['notify_yaml_filepath'] arg['notify.yaml'] = SupportFile.read_file(arg['notify_yaml_filepath']) + elif page == 'celery': + arg['use_celery'] = F.config['use_celery'] + arg['running_type'] = F.config['running_type'] return render_template(f'{__package__}_{name}_{page}.html', arg=arg) except Exception as e: @@ -113,41 +121,52 @@ class ModuleSetting(PluginModuleBase): elif command == 'command_run': ret['msg'] = arg1 pass + elif command == 'celery_execute': + tmp = arg1.replace("{F.config['config_filepath']}", F.config['config_filepath']).replace('{F.config["config_filepath"]}', F.config['config_filepath']) + cmd = [ + ['msg', f'명령 : {tmp}'], + ['msg', ''], + tmp.split(' '), + ] + ToolModalCommand.start("Celery 실행", cmd) return jsonify(ret) def plugin_load(self): try: - if F.config['run_flask']: - F.logger.info(f"arg_repeat : {F.config['arg_repeat']}") - F.logger.info(f"arg_repeat : {F.config['arg_repeat']}") + if F.config['run_flask'] == False: + return + F.logger.info(f"arg_repeat : {F.config['arg_repeat']}") + F.logger.info(f"arg_repeat : {F.config['arg_repeat']}") - if F.config['arg_repeat'] == 0 or SystemModelSetting.get('system_start_time') == '': - SystemModelSetting.set('system_start_time', datetime.now().strftime('%Y-%m-%d %H:%M:%S')) - SystemModelSetting.set('repeat', str(F.config['arg_repeat'])) - username = SystemModelSetting.get('web_id') - passwd = SystemModelSetting.get('web_pw') - F.users[username] = User(username, passwd_hash=passwd) + if F.config['arg_repeat'] == 0 or SystemModelSetting.get('system_start_time') == '': + SystemModelSetting.set('system_start_time', datetime.now().strftime('%Y-%m-%d %H:%M:%S')) + SystemModelSetting.set('repeat', str(F.config['arg_repeat'])) + username = SystemModelSetting.get('web_id') + passwd = SystemModelSetting.get('web_pw') + F.users[username] = User(username, passwd_hash=passwd) - self.__set_restart_scheduler() - self.__set_scheduler_check_scheduler() - F.get_recent_version() + self.__set_restart_scheduler() + self.__set_scheduler_check_scheduler() + F.get_recent_version() - notify_yaml_filepath = os.path.join(F.config['path_data'], 'db', 'notify.yaml') - if os.path.exists(notify_yaml_filepath) == False: - import shutil - shutil.copy( - os.path.join(F.config['path_app'], 'files', 'notify.yaml.template'), - notify_yaml_filepath - ) - if SystemModelSetting.get_bool('restart_notify'): - from tool import ToolNotify - msg = f"시스템이 시작되었습니다.\n재시작: {F.config['arg_repeat']}" - ToolNotify.send_message(msg, message_id='system_start') + notify_yaml_filepath = os.path.join(F.config['path_data'], 'db', 'notify.yaml') + if os.path.exists(notify_yaml_filepath) == False: + import shutil + shutil.copy( + os.path.join(F.config['path_app'], 'files', 'notify.yaml.template'), + notify_yaml_filepath + ) + if SystemModelSetting.get_bool('restart_notify'): + from tool import ToolNotify + msg = f"시스템이 시작되었습니다.\n재시작: {F.config['arg_repeat']}" + ToolNotify.send_message(msg, message_id='system_start') except Exception as e: P.logger.error(f'Exception:{str(e)}') P.logger.error(traceback.format_exc()) + def plugin_unload(self): + ToolModalCommand.process_close() def setting_save_after(self, change_list): if 'theme' in change_list: diff --git a/lib/system/mod_tool.py b/lib/system/mod_tool.py index 74a100d..2eab1b3 100644 --- a/lib/system/mod_tool.py +++ b/lib/system/mod_tool.py @@ -11,7 +11,7 @@ class ModuleTool(PluginModuleBase): def __init__(self, P): - super(ModuleTool, self).__init__(P, name=name, first_menu='celery') + super(ModuleTool, self).__init__(P, name=name, first_menu='upload') def process_menu(self, page, req): diff --git a/lib/system/setup.py b/lib/system/setup.py index 9b8fd57..08ec564 100644 --- a/lib/system/setup.py +++ b/lib/system/setup.py @@ -12,6 +12,7 @@ __menu = { {'uri': 'menu', 'name': '메뉴 구성'}, {'uri': 'config', 'name': 'config.yaml 파일'}, {'uri': 'export', 'name': 'export.sh 파일'}, + {'uri': 'celery', 'name': '비동기 작업(celery)'}, {'uri': 'notify', 'name': '알림'}, ], @@ -22,6 +23,7 @@ __menu = { 'list': [ {'uri': 'setting', 'name': '설정'}, {'uri': 'list', 'name': '로딩 플러그인'}, + {'uri': 'all', 'name': '플러그인 목록'}, ], }, { @@ -29,7 +31,6 @@ __menu = { 'name': '시스템 툴', 'list': [ {'uri': 'upload', 'name': '업로드'}, - {'uri': 'celery', 'name': 'celery 테스트'}, {'uri': 'python', 'name': 'Python'}, {'uri': 'db', 'name': 'DB'}, {'uri': 'crypt', 'name': '암호화'}, diff --git a/lib/system/templates/system_setting_celery.html b/lib/system/templates/system_setting_celery.html new file mode 100644 index 0000000..5a1843c --- /dev/null +++ b/lib/system/templates/system_setting_celery.html @@ -0,0 +1,123 @@ +{% extends "base.html" %} +{% block content %} +
+ {{ macros.m_button_group([['globalSettingSaveBtn', '설정 저장'], ['celery_test_btn', 'Celery 테스트']])}} + {{ macros.m_row_start('5') }} + {{ macros.m_row_end() }} + {{ macros.m_hr() }} + + +
+ + + +{% endblock %} diff --git a/lib/system/templates/system_tool_crypt.html b/lib/system/templates/system_tool_crypt.html index 71be10d..4d040dc 100644 --- a/lib/system/templates/system_tool_crypt.html +++ b/lib/system/templates/system_tool_crypt.html @@ -13,7 +13,7 @@