codekingpro/portable-devtools
115k
1# -*- coding: utf-8 -*-2 3##########################################################################4#5# pgAdmin 4 - PostgreSQL Tools6#7# Copyright (C) 2013 - 2024, The pgAdmin Development Team8# This software is released under the PostgreSQL License9#10##########################################################################11 12"""13This python script is responsible for executing a process, and logs its output,14and error in the given output directory.15 16We will create a detached process, which executes this script.17 18This script will:19* Fetch the configuration from the given database.20* Run the given executable specified in the configuration with the arguments.21* Create log files for both stdout, and stdout.22* Update the start time, end time, exit code, etc in the configuration23 database.24 25Args:26 list of program and arguments passed to it.27 28It also depends on the following environment variable for proper execution.29PROCID - Process-id30OUTDIR - Output directory31"""32 33# To make print function compatible with python2 & python334import sys35import os36from datetime import datetime, timedelta, tzinfo, timezone37from subprocess import Popen, PIPE38from threading import Thread39import signal40 41_IS_WIN = (os.name == 'nt')42_ZERO = timedelta(0)43_sys_encoding = None44_fs_encoding = None45_out_dir = None46_log_file = None47 48 49def _log(msg):50 with open(_log_file, 'a') as fp:51 fp.write(52 ('INFO:: %s\n' % msg.encode('ascii', 'xmlcharrefreplace'))53 )54 55 56def unescape_dquotes_process_arg(arg):57 # Double quotes has special meaning for shell command line and they are58 # run without the double quotes.59 #60 # Remove the saviour #DQ#61 62 # This cannot be at common place as this file executes63 # separately from pgadmin64 dq_id = "#DQ#"65 66 if arg.startswith(dq_id) and arg.endswith(dq_id):67 return '{0}'.format(arg[len(dq_id):-len(dq_id)])68 else:69 return arg70 71 72def _log_exception():73 type_, value_, traceback_ = sys.exc_info()74 75 with open(_log_file, 'a') as fp:76 from traceback import format_exception77 res = ''.join(78 format_exception(type_, value_, traceback_)79 )80 81 fp.write('EXCEPTION::\n{0}'.format(res))82 return res83 84 85# Copied the 'UTC' class from the 'pytz' package to allow to run this script86# without any external dependent library, and can be used with any python87# version.88class UTC(tzinfo):89 """UTC90 91 Optimized UTC implementation. It unpickles using the single module global92 instance defined beneath this class declaration.93 """94 zone = "UTC"95 96 _utcoffset = _ZERO97 _dst = _ZERO98 _tzname = zone99 100 def fromutc(self, dt):101 if dt.tzinfo is None:102 return self.localize(dt)103 return super(UTC.__class__, self).fromutc(dt)104 105 def utcoffset(self, dt):106 return _ZERO107 108 def tzname(self, dt):109 return "UTC"110 111 def dst(self, dt):112 return _ZERO113 114 def localize(self, dt):115 """Convert naive time to local time"""116 if dt.tzinfo is not None:117 raise ValueError('Not naive datetime (tzinfo is already set)')118 return dt.replace(tzinfo=self)119 120 def normalize(self, dt):121 """Correct the timezone information on the given datetime"""122 if dt.tzinfo is self:123 return dt124 if dt.tzinfo is None:125 raise ValueError('Naive time - no tzinfo set')126 return dt.astimezone(self)127 128 def __repr__(self):129 return "<UTC>"130 131 def __str__(self):132 return "UTC"133 134 135def get_current_time(format='%Y-%m-%d %H:%M:%S.%f %z'):136 return datetime.now(timezone.utc).strftime(format)137 138 139class ProcessLogger(Thread):140 """141 This class definition is responsible for capturing & logging142 stdout & stderr messages from subprocess143 144 Methods:145 --------146 * __init__(stream_type)147 - This method is use to initlize the ProcessLogger class object148 149 * log(msg)150 - Log message in the orderly manner.151 152 * run()153 - Reads the stdout/stderr for messages and sent them to logger154 """155 156 def __init__(self, stream_type):157 """158 This method is use to initialize the ProcessLogger class object159 160 Args:161 stream_type: Type of STD (std)162 163 Returns:164 None165 """166 import codecs167 168 Thread.__init__(self)169 self.process = None170 self.stream = None171 self.logger = open(os.path.join(_out_dir, stream_type), 'wb',172 buffering=0)173 174 def attach_process_stream(self, process, stream):175 """176 This function will attach a process and its stream with this thread.177 178 Args:179 process: Process180 stream: Stream attached with the process181 182 Returns:183 None184 """185 self.process = process186 self.stream = stream187 188 def log(self, msg):189 """190 This function will update log file191 192 Args:193 msg: message194 195 Returns:196 None197 """198 # Write into log file199 if self.logger:200 if msg:201 self.logger.write(202 get_current_time(203 format='%y%m%d%H%M%S%f'204 ).encode('utf-8')205 )206 self.logger.write(b',')207 self.logger.write(208 msg.lstrip(b'\r\n' if _IS_WIN else b'\n')209 )210 self.logger.write(os.linesep.encode('utf-8'))211 212 return True213 return False214 215 def run(self):216 if self.process and self.stream:217 while True:218 nextline = self.stream.readline()219 220 if nextline:221 self.log(nextline)222 else:223 if self.process.poll() is not None:224 break225 226 def release(self):227 if self.logger:228 self.logger.close()229 self.logger = None230 231 232def update_status(**kw):233 """234 This function will updates process stats235 236 Args:237 kwargs - Process configuration details238 239 Returns:240 None241 """242 import json243 244 if _out_dir:245 status = dict(246 (k, v) for k, v in kw.items()247 if k in ('start_time', 'end_time', 'exit_code', 'pid')248 )249 _log('Updating the status:\n{0}'.format(json.dumps(status)))250 with open(os.path.join(_out_dir, 'status'), 'w') as fp:251 json.dump(status, fp)252 else:253 raise ValueError("Please verify pid and db_file arguments.")254 255 256def _handle_execute_exception(ex, args, _stderr, exit_code=None):257 """258 Used internally by execute to handle exception259 :param ex: exception object260 :param args: execute args dict261 :param _stderr: stderr262 :param exit_code: exit code override263 """264 info = _log_exception()265 if _stderr:266 _stderr.log(info)267 else:268 print('WARNING: {0}'.format(str(ex)))269 args.update({'end_time': get_current_time()})270 args.update({271 'exit_code': ex.errno if exit_code is None else exit_code})272 273 274def _fetch_execute_output(process, _stdout, _stderr):275 """276 Used internally by execute to fetch execute output and log it.277 :param process: process obj278 :param _stdout: stdout279 :param _stderr: stderr280 """281 data = process.communicate()282 if data:283 if data[0]:284 _stdout.log(data[0])285 if data[1]:286 _stderr.log(data[1])287 288 289def execute(argv):290 """291 This function will execute the background process292 293 Returns:294 None295 """296 command = argv[1:]297 args = dict()298 _log('Initialize the process execution: {0}'.format(command))299 300 # Create seprate thread for stdout and stderr301 process_stdout = ProcessLogger('out')302 process_stderr = ProcessLogger('err')303 304 try:305 # update start_time306 args.update({307 'start_time': get_current_time(),308 'stdout': process_stdout.log,309 'stderr': process_stderr.log,310 'pid': os.getpid()311 })312 313 # Update start time314 update_status(**args)315 _log('Status updated...')316 317 if os.environ.get(os.environ.get('PROCID', None), None):318 os.environ['PGPASSWORD'] = os.environ[os.environ['PROCID']]319 320 kwargs = dict()321 kwargs['close_fds'] = False322 kwargs['shell'] = True if _IS_WIN else False323 324 # We need environment variables & values in string325 kwargs['env'] = os.environ.copy()326 327 _log('Starting the command execution...')328 process = Popen(329 command, stdout=PIPE, stderr=PIPE, stdin=None, **kwargs330 )331 args.update({332 'start_time': get_current_time(),333 'stdout': process_stdout.log,334 'stderr': process_stderr.log,335 'pid': process.pid336 })337 update_status(**args)338 _log('Status updated after starting child process...')339 340 _log('Attaching the loggers to stdout, and stderr...')341 # Attach the stream to the process logger, and start logging.342 process_stdout.attach_process_stream(process, process.stdout)343 process_stdout.start()344 process_stderr.attach_process_stream(process, process.stderr)345 process_stderr.start()346 347 # Join both threads together348 process_stdout.join()349 process_stderr.join()350 351 _log('Waiting for the process to finish...')352 # Child process return code353 exit_code = process.wait()354 355 if exit_code is None:356 exit_code = process.poll()357 358 _log('Process exited with code: {0}'.format(exit_code))359 args.update({'exit_code': exit_code})360 361 # Add end_time362 args.update({'end_time': get_current_time()})363 364 # Fetch last output, and error from process if it has missed.365 _fetch_execute_output(process, process_stdout, process_stderr)366 367 # If executable not found or invalid arguments passed368 except OSError as e:369 _handle_execute_exception(e, args, process_stderr, exit_code=None)370 # Unknown errors371 except Exception as e:372 _handle_execute_exception(e, args, process_stderr, exit_code=-1)373 finally:374 # Update the execution end_time, and exit-code.375 update_status(**args)376 _log('Exiting the process executor...')377 if process_stderr:378 process_stderr.release()379 if process_stdout:380 process_stdout.release()381 _log('Bye!')382 383 384def signal_handler(signal, msg):385 # Let's ignore all the signal comming to us.386 pass387 388 389def convert_environment_variables(env):390 """391 This function is use to convert environment variable to string392 because environment variable must be string in popen393 :param env: Dict of environment variable394 :return: Encoded environment variable as string395 """396 temp_env = dict()397 for key, value in env.items():398 try:399 if not isinstance(key, str):400 key = key.encode(_sys_encoding)401 if not isinstance(value, str):402 value = value.encode(_sys_encoding)403 temp_env[key] = value404 except Exception:405 _log_exception()406 return temp_env407 408 409if __name__ == '__main__':410 411 argv = [412 unescape_dquotes_process_arg(arg) for arg in sys.argv413 ]414 415 _sys_encoding = sys.getdefaultencoding()416 if not _sys_encoding or _sys_encoding == 'ascii':417 # Fall back to 'utf-8', if we couldn't determine the default encoding,418 # or 'ascii'.419 _sys_encoding = 'utf-8'420 421 _fs_encoding = sys.getfilesystemencoding()422 if not _fs_encoding or _fs_encoding == 'ascii':423 # Fall back to 'utf-8', if we couldn't determine the file-system424 # encoding or 'ascii'.425 _fs_encoding = 'utf-8'426 427 _out_dir = os.environ['OUTDIR']428 _log_file = os.path.join(_out_dir, ('log_%s' % os.getpid()))429 430 _log('Starting the process executor...')431 432 # Ignore any signals433 signal.signal(signal.SIGINT, signal_handler)434 signal.signal(signal.SIGTERM, signal_handler)435 _log('Disabled the SIGINT, SIGTERM signals...')436 437 if _IS_WIN:438 _log('Disable the SIGBREAKM signal (windows)...')439 signal.signal(signal.SIGBREAK, signal_handler)440 _log('Disabled the SIGBREAKM signal (windows)...')441 442 # For windows:443 # We would run the process_executor in the detached mode again to make444 # the child process to run as a daemon. And, it would run without445 # depending on the status of the web-server.446 if 'PGA_BGP_FOREGROUND' in os.environ and \447 os.environ['PGA_BGP_FOREGROUND'] == "1":448 _log('[CHILD] Start process execution...')449 # This is a child process running as the daemon process.450 # Let's do the job assigning to it.451 try:452 _log('Executing the command now from the detached child...')453 execute(argv)454 except Exception:455 _log_exception()456 else:457 from subprocess import CREATE_NEW_PROCESS_GROUP458 DETACHED_PROCESS = 0x00000008459 460 # Forward the standard input, output, and error stream to the461 # 'devnull'.462 stdin = open(os.devnull, "r")463 stdout = open(os.devnull, "a")464 stderr = open(os.devnull, "a")465 env = os.environ.copy()466 env['PGA_BGP_FOREGROUND'] = "1"467 468 # We need environment variables & values in string469 _log('[PARENT] Converting the environment variable in the '470 'bytes format...')471 try:472 env = convert_environment_variables(env)473 except Exception:474 _log_exception()475 476 kwargs = {477 'stdin': stdin.fileno(),478 'stdout': stdout.fileno(),479 'stderr': stderr.fileno(),480 'creationflags': CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS,481 'close_fds': False,482 'cwd': _out_dir,483 'env': env484 }485 486 cmd = [sys.executable]487 cmd.extend(argv)488 489 _log('[PARENT] Command executings: {0}'.format(cmd))490 491 p = Popen(cmd, **kwargs)492 493 exitCode = p.poll()494 495 if exitCode is not None:496 _log(497 '[PARENT] Child exited with exit-code#{0}...'.format(498 exitCode499 )500 )501 else:502 _log('[PARENT] Started the child with PID#{0}'.format(p.pid))503 504 # Question: Should we wait for sometime?505 # Answer: Looks the case...506 from time import sleep507 sleep(2)508 _log('[PARENT] Exiting...')509 sys.exit(0)510 else:511 r, w = os.pipe()512 513 # For POSIX:514 # We will fork the process, and run the child process as daemon, and515 # let it do the job.516 if os.fork() == 0:517 _log('[CHILD] Forked the child process...')518 # Hmm... So - I need to do the job now...519 try:520 os.close(r)521 522 _log('[CHILD] Make the child process leader...')523 # Let me be the process leader first.524 os.setsid()525 os.umask(0)526 527 _log('[CHILD] Make the child process leader...')528 w = os.fdopen(w, 'w')529 # Let me inform my parent - I will do the job, do not worry530 # now, and die peacefully.531 _log('[CHILD] Inform parent about successful child forking...')532 w.write('1')533 w.close()534 535 _log('[CHILD] Start executing the background process...')536 execute(argv)537 except Exception:538 _log_exception()539 sys.exit(1)540 else:541 os.close(w)542 r = os.fdopen(r)543 # I do not care, what the child send.544 r.read()545 _log('[PARENT] Got message from the child...')546 r.close()547 548 _log('[PARENT] Exiting...')549 sys.exit(0)550