454 lines
17 KiB
Python
454 lines
17 KiB
Python
import logging
|
|
import logging.handlers
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from datetime import datetime
|
|
|
|
from flask import Flask
|
|
from flask_cors import CORS
|
|
from flask_login import LoginManager, login_required
|
|
from flask_socketio import SocketIO
|
|
from flask_sqlalchemy import SQLAlchemy
|
|
from flaskext.markdown import Markdown
|
|
from pytz import timezone, utc
|
|
|
|
from .init_declare import CustomFormatter, check_api
|
|
|
|
|
|
class Framework:
|
|
__instance = None
|
|
|
|
@classmethod
|
|
def get_instance(cls):
|
|
if cls.__instance == None:
|
|
cls.__instance = Framework()
|
|
return cls.__instance
|
|
|
|
|
|
def __init__(self):
|
|
self.logger = None
|
|
self.app = None
|
|
self.celery = None
|
|
self.db = None
|
|
self.scheduler = None
|
|
self.socketio = None
|
|
self.path_app_root = None
|
|
self.path_data = None
|
|
self.users = {}
|
|
|
|
self.__level_unset_logger_list = []
|
|
self.__logger_list = []
|
|
self.__exit_code = -1
|
|
self.login_manager = None
|
|
#self.plugin_instance_list = {}
|
|
#self.plugin_menus = {}
|
|
|
|
# 그냥 F. 로 접근 하게....
|
|
self.SystemModelSetting = None
|
|
self.Job = None
|
|
self.login_required = login_required
|
|
self.check_api = check_api
|
|
self.__initialize()
|
|
|
|
|
|
def __initialize(self):
|
|
self.__config_initialize("first")
|
|
self.__make_default_dir()
|
|
|
|
self.logger = self.get_logger(__package__)
|
|
|
|
from support import set_logger
|
|
set_logger(self.logger)
|
|
|
|
self.__prepare_starting()
|
|
self.app = Flask(__name__)
|
|
self.__config_initialize('flask')
|
|
|
|
self.db = SQLAlchemy(self.app, session_options={"autoflush": False})
|
|
|
|
if True or self.config['run_flask']:
|
|
from .scheduler import Job, Scheduler
|
|
self.scheduler = Scheduler(self)
|
|
self.Job = Job
|
|
|
|
if self.config['use_gevent']:
|
|
self.socketio = SocketIO(self.app, cors_allowed_origins="*")
|
|
else:
|
|
self.socketio = SocketIO(self.app, cors_allowed_origins="*", async_mode='threading')
|
|
|
|
CORS(self.app)
|
|
Markdown(self.app)
|
|
|
|
self.login_manager = LoginManager()
|
|
self.login_manager.init_app(self.app)
|
|
self.login_manager.login_view = "/system/login"
|
|
|
|
self.celery = self.__init_celery()
|
|
|
|
|
|
def __init_celery(self):
|
|
try:
|
|
from celery import Celery
|
|
|
|
#if frame.config['use_celery'] == False or platform.system() == 'Windows':
|
|
if self.config['use_celery'] == False:
|
|
raise Exception('no celery')
|
|
try:
|
|
redis_port = os.environ['REDIS_PORT']
|
|
except:
|
|
try:
|
|
redis_port = self.config['redis_port']
|
|
except:
|
|
redis_port = '6379'
|
|
|
|
self.app.config['CELERY_BROKER_URL'] = 'redis://localhost:%s/0' % redis_port
|
|
self.app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost:%s/0' % redis_port
|
|
celery = Celery(self.app.name, broker=self.app.config['CELERY_BROKER_URL'], backend=self.app.config['CELERY_RESULT_BACKEND'])
|
|
celery.conf['CELERY_ENABLE_UTC'] = False
|
|
celery.conf.update(
|
|
task_serializer='pickle',
|
|
result_serializer='pickle',
|
|
accept_content=['pickle'],
|
|
timezone='Asia/Seoul'
|
|
)
|
|
from celery import bootsteps
|
|
from click import Option
|
|
celery.user_options['worker'].add(Option(('--config_filepath',), help=''),)
|
|
celery.user_options['worker'].add(Option(('--running_type',), help=''),)
|
|
class CustomArgs(bootsteps.Step):
|
|
def __init__(self, worker, config_filepath=None, running_type=None, **options):
|
|
from . import F
|
|
F.logger.info(f"celery config_filepath: {config_filepath}")
|
|
F.logger.info(f"celery running_type: {running_type}")
|
|
#F.logger.info(f"celery running_type: {options}")
|
|
celery.steps['worker'].add(CustomArgs)
|
|
except Exception as e:
|
|
self.logger.error('CELERY!!!')
|
|
self.logger.error(f'Exception:{str(e)}')
|
|
self.logger.error(traceback.format_exc())
|
|
def dummy_func():
|
|
pass
|
|
|
|
class celery(object):
|
|
class task(object):
|
|
def __init__(self, *args, **kwargs):
|
|
if len(args) > 0:
|
|
self.f = args[0]
|
|
|
|
def __call__(self, *args, **kwargs):
|
|
if len(args) > 0 and type(args[0]) == type(dummy_func):
|
|
return args[0]
|
|
self.f(*args, **kwargs)
|
|
return celery
|
|
|
|
|
|
def initialize_system(self):
|
|
from system.setup import P
|
|
SystemInstance = P
|
|
try:
|
|
self.db.create_all()
|
|
except Exception as e:
|
|
self.logger.error('CRITICAL db.create_all()!!!')
|
|
self.logger.error(f'Exception:{str(e)}')
|
|
self.logger.error(traceback.format_exc())
|
|
SystemInstance.plugin_load()
|
|
self.app.register_blueprint(SystemInstance.blueprint)
|
|
self.config['flag_system_loading'] = True
|
|
self.__config_initialize('member')
|
|
self.__config_initialize('system_loading_after')
|
|
self.SystemModelSetting = SystemInstance.ModelSetting
|
|
|
|
|
|
def initialize_plugin(self):
|
|
from system.setup import P as SP
|
|
|
|
from .init_web import jinja_initialize
|
|
jinja_initialize(self.app)
|
|
|
|
#system.LogicPlugin.custom_plugin_update()
|
|
from .init_plugin import PluginManager
|
|
self.PluginManager = PluginManager
|
|
PluginManager.plugin_init()
|
|
PluginManager.plugin_menus['system'] = {'menu':SP.menu, 'match':False}
|
|
|
|
#from .init_menu import init_menu, get_menu_map
|
|
from .init_menu import MenuManager
|
|
MenuManager.init_menu()
|
|
#init_menu(self.plugin_menu)
|
|
#system.SystemLogic.apply_menu_link()
|
|
|
|
if self.config['run_flask']:
|
|
if self.config.get('port') == None:
|
|
self.config['port'] = SP.SystemModelSetting.get_int('port')
|
|
|
|
from . import init_route, log_viewer
|
|
|
|
self.__make_default_logger()
|
|
self.logger.info('### LAST')
|
|
self.logger.info(f"### PORT: {self.config['port']}")
|
|
self.logger.info('### Now you can access App by webbrowser!!')
|
|
|
|
|
|
def __prepare_starting(self):
|
|
# 여기서 monkey.patch시 너무 늦다고 문제 발생
|
|
pass
|
|
|
|
|
|
###################################################
|
|
# 환경
|
|
###################################################
|
|
def __config_initialize(self, mode):
|
|
if mode == "first":
|
|
self.config = {}
|
|
self.config['os'] = platform.system()
|
|
self.config['flag_system_loading'] = False
|
|
self.config['run_flask'] = True if sys.argv[0].endswith('main.py') else False
|
|
self.config['run_celery'] = True if sys.argv[0].find('celery') != -1 else False
|
|
|
|
self.config['path_app'] = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
if self.config['os'] == 'Windows' and self.config['path_app'][0] != '/':
|
|
self.config['path_app'] = self.config['path_app'][0].upper() + self.config['path_app'][1:]
|
|
self.path_app_root = self.config['path_app']
|
|
self.config['path_working'] = os.getcwd()
|
|
if os.environ.get('RUNNING_TYPE') == 'docker':
|
|
self.config['running_type'] = 'docker'
|
|
self.__process_args()
|
|
self.__load_config()
|
|
self.__init_define()
|
|
elif mode == "flask":
|
|
self.app.secret_key = os.urandom(24)
|
|
#self.app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///data/db/system.db?check_same_thread=False'
|
|
self.app.config['SQLALCHEMY_BINDS'] = {}
|
|
self.app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
|
|
self.app.config['TEMPLATES_AUTO_RELOAD'] = True
|
|
self.app.config['JSON_AS_ASCII'] = False
|
|
elif mode == 'system_loading_after':
|
|
if 'running_type' not in self.config:
|
|
self.config['running_type'] = 'native'
|
|
|
|
|
|
def __init_define(self):
|
|
self.config['DEFINE'] = {}
|
|
# 이건 필요 없음
|
|
self.config['DEFINE']['MAIN_SERVER_URL'] = 'https://server.sjva.me'
|
|
|
|
|
|
def __process_args(self):
|
|
# celery 에서 args 처리시 문제 발생.
|
|
if self.config['run_flask']:
|
|
import argparse
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--config', default='.', help='config filepath. Default: {current folder}/config.yaml')
|
|
parser.add_argument('--repeat', default=0, type=int, help=u'Do not set. This value is set by automatic')
|
|
args = parser.parse_args()
|
|
self.config['arg_repeat'] = args.repeat
|
|
self.config['arg_config'] = args.config
|
|
else:
|
|
# 아주 안좋은 구조..
|
|
# celery user_options으로 configfilepath를 받은 후 처리해야하나, 로그파일 경로 등에서 데이터 폴더 위치를 미리 사용하는 경우가 많다.
|
|
# sys.argv에서 데이터 경로를 바로 가져와서 사용.
|
|
self.config['arg_repeat'] = 0
|
|
#self.config['arg_config'] = sys.argv[-1].split('=')[-1]
|
|
#self.config['arg_config'] = sys.argv[-1].split('=')[-1]
|
|
for tmp in sys.argv:
|
|
if tmp.startswith('--config_filepath'):
|
|
self.config['arg_config'] = tmp.split('=')[1]
|
|
#break
|
|
elif tmp.startswith('--running_type'):
|
|
self.config['running_type'] = tmp.split('=')[1]
|
|
|
|
#self.config['arg_config'] =
|
|
|
|
def __load_config(self):
|
|
from .init_declare import read_yaml
|
|
|
|
#if self.config['run_flask']:
|
|
if self.config['arg_config'] == '.':
|
|
#self.config['config_filepath'] = os.path.join(self.path_app_root, 'config.yaml')
|
|
self.config['config_filepath'] = os.path.join(self.config['path_working'], 'config.yaml')
|
|
else:
|
|
self.config['config_filepath'] = self.config['arg_config']
|
|
if not os.path.exists(self.config['config_filepath']):
|
|
# celery는 환경변수 사용불가로 native 판단
|
|
# 도커는 celery가 먼저 진입
|
|
# 추후에 변경할 것!!!!!!!!!!!!!!!!! TODO
|
|
#if self.config.get('running_type') == 'docker':
|
|
if self.config.get('running_type') == 'docker':# or os.path.exists('/data'):
|
|
shutil.copy(
|
|
os.path.join(self.path_app_root, 'files', 'config.yaml.docker'),
|
|
self.config['config_filepath']
|
|
)
|
|
else:
|
|
shutil.copy(
|
|
os.path.join(self.path_app_root, 'files', 'config.yaml.template'),
|
|
self.config['config_filepath']
|
|
)
|
|
data = read_yaml(self.config['config_filepath'])
|
|
for key, value in data.items():
|
|
if key == 'running_type' and value not in ['termux', 'entware']:
|
|
continue
|
|
self.config[key] = value
|
|
|
|
if self.config['path_data'] == '.':
|
|
self.config['path_data'] = self.config['path_working']
|
|
# 예외적으로 현재폴더가 app일 경우 지저분해지는 것을 방지하기 위해 data 로 지정
|
|
if self.config['path_data'] == self.config['path_working']:
|
|
self.config['path_data'] = os.path.join(self.config['path_working'], 'data')
|
|
self.path_data = self.config['path_data']
|
|
|
|
|
|
|
|
def __make_default_dir(self):
|
|
os.makedirs(self.config['path_data'], exist_ok=True)
|
|
tmp = os.path.join(self.config['path_data'], 'tmp')
|
|
try:
|
|
import shutil
|
|
if os.path.exists(tmp):
|
|
shutil.rmtree(tmp)
|
|
except:
|
|
pass
|
|
sub = ['db', 'log', 'tmp']
|
|
for item in sub:
|
|
tmp = os.path.join(self.config['path_data'], item)
|
|
os.makedirs(tmp, exist_ok=True)
|
|
|
|
###################################################
|
|
|
|
|
|
|
|
|
|
###################################################
|
|
# 로그
|
|
###################################################
|
|
def get_logger(self, name):
|
|
logger = logging.getLogger(name)
|
|
if not logger.handlers:
|
|
level = logging.DEBUG
|
|
try:
|
|
if self.config['flag_system_loading']:
|
|
try:
|
|
from system import SystemModelSetting
|
|
level = SystemModelSetting.get_int('log_level')
|
|
except:
|
|
level = logging.DEBUG
|
|
if self.__level_unset_logger_list is not None:
|
|
for item in self.__level_unset_logger_list:
|
|
item.setLevel(level)
|
|
self.__level_unset_logger_list = None
|
|
else:
|
|
self.__level_unset_logger_list.append(logger)
|
|
if name.startswith('apscheduler'):
|
|
level = logging.CRITICAL
|
|
else:
|
|
self.__logger_list.append(logger)
|
|
except:
|
|
pass
|
|
logger.setLevel(level)
|
|
file_formatter = logging.Formatter(u'[%(asctime)s|%(levelname)s|%(filename)s:%(lineno)s] %(message)s')
|
|
def customTime(*args):
|
|
utc_dt = utc.localize(datetime.utcnow())
|
|
my_tz = timezone("Asia/Seoul")
|
|
converted = utc_dt.astimezone(my_tz)
|
|
return converted.timetuple()
|
|
|
|
file_formatter.converter = customTime
|
|
file_max_bytes = 1 * 1024 * 1024
|
|
fileHandler = logging.handlers.RotatingFileHandler(filename=os.path.join(self.path_data, 'log', f'{name}.log'), maxBytes=file_max_bytes, backupCount=5, encoding='utf8', delay=True)
|
|
streamHandler = logging.StreamHandler()
|
|
|
|
# handler에 fommater 세팅
|
|
fileHandler.setFormatter(file_formatter)
|
|
streamHandler.setFormatter(CustomFormatter())
|
|
|
|
# Handler를 logging에 추가
|
|
logger.addHandler(fileHandler)
|
|
logger.addHandler(streamHandler)
|
|
return logger
|
|
|
|
|
|
def __make_default_logger(self):
|
|
self.get_logger('apscheduler.scheduler')
|
|
self.get_logger('apscheduler.executors.default')
|
|
try: logging.getLogger('socketio').setLevel(logging.ERROR)
|
|
except: pass
|
|
try: logging.getLogger('engineio').setLevel(logging.ERROR)
|
|
except: pass
|
|
try: logging.getLogger('apscheduler.scheduler').setLevel(logging.ERROR)
|
|
except: pass
|
|
try: logging.getLogger('apscheduler.executors.default').setLevel(logging.ERROR)
|
|
except: pass
|
|
try: logging.getLogger('werkzeug').setLevel(logging.ERROR)
|
|
except: pass
|
|
|
|
def set_level(self, level):
|
|
try:
|
|
for l in self.__logger_list:
|
|
l.setLevel(level)
|
|
self.__make_default_logger()
|
|
except:
|
|
pass
|
|
###################################################
|
|
|
|
|
|
def start(self):
|
|
host = '0.0.0.0'
|
|
for i in range(10):
|
|
try:
|
|
#self.logger.debug(d(self.config))
|
|
self.socketio.run(self.app, host=host, port=self.config['port'], debug=self.config['debug'], use_reloader=self.config['use_reloader'])
|
|
self.logger.warning(f"EXIT CODE : {self.__exit_code}")
|
|
# 2021-05-18
|
|
if self.config['running_type'] in ['termux', 'entware']:
|
|
os._exit(self.__exit_code)
|
|
else:
|
|
if self.__exit_code != -1:
|
|
sys.exit(self.__exit_code)
|
|
else:
|
|
self.logger.warning(f"framework.exit_code is -1")
|
|
break
|
|
except Exception as exception:
|
|
self.logger.error(f"Start ERROR : {str(exception)}")
|
|
host = '127.0.0.1'
|
|
time.sleep(10*i)
|
|
continue
|
|
except KeyboardInterrupt:
|
|
self.logger.error('KeyboardInterrupt !!')
|
|
#except SystemExit:
|
|
# return
|
|
#sys.exit(self.__exit_code)
|
|
|
|
|
|
# system 플러그인에서 콜
|
|
def restart(self):
|
|
self.__exit_code = 1
|
|
self.__app_close()
|
|
|
|
def shutdown(self):
|
|
self.__exit_code = 0
|
|
self.__app_close()
|
|
|
|
def __app_close(self):
|
|
try:
|
|
from .init_plugin import PluginManager
|
|
PluginManager.plugin_unload()
|
|
self.socketio.stop()
|
|
except Exception as exception:
|
|
self.logger.error('Exception:%s', exception)
|
|
self.logger.error(traceback.format_exc())
|
|
|
|
def get_recent_version(self):
|
|
try:
|
|
import requests
|
|
url = f"{self.config['DEFINE']['MAIN_SERVER_URL']}/version"
|
|
self.config['recent_version'] = requests.get(url).text
|
|
return True
|
|
except Exception as e:
|
|
self.logger.error(f'Exception:{str(e)}')
|
|
self.logger.error(traceback.format_exc())
|
|
self.config['recent_version'] = '확인 실패'
|
|
return False
|