Tryag File Manager
Home
||
Turbo Force
||
B-F Config_Cpanel
Current Path :
/
paip
/
script
/
weight
/
Or
Select Your Path :
Upload File :
New :
File
Dir
//paip/script/weight/dailyDBupdator_backup.py
import os, sys from datetime import date, timedelta import pandas as pd import numpy as np import datetime from scipy import stats # set sys path to import PyDBconnector HOME_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir) DATA_DIR = 'data' MODULE_DIR = 'util' OUTPUT_DIR = 'out' isUp = 50. insertDB = 1 sys.path.append(os.path.join(HOME_PATH, MODULE_DIR)) from PyDBconnector import PyDBconnector from utils import roundUp from itertools import chain import warnings warnings.filterwarnings('ignore') def apply_fn1(x): tempVal = list() # print(f"{type(x['predPixelMean'])}, value : {x['predPixelMean'].values}") tempVal.append(x['predPixelMean'].values.tolist()) return pd.DataFrame({'predCount': [sum(x['predCount'].values)], \ 'predPixelMean': tempVal, \ 'noChickCount': [sum(x['noChickCount'].values)], \ 'noChickMean': [sum(x['noChickMean'].values)], \ 'meanWeight': ','.join([str(round(y, 1)) for y in x['meanWeight']]) \ }) def fn_roll_transform(x): df = pd.DataFrame(x) return df.iloc[:, 0].rolling(2).apply(lambda x: x.iloc[1] - x.iloc[0]) def fn_shift_transform(x): df = pd.DataFrame(x) return df.iloc[:, 0].shift(1) if __name__ == '__main__': dbconn = PyDBconnector() yesterday = date.today() - timedelta(1) # yesterday='2022-01-08' yesterday_str = f'''select * from tbl_weight_stats ''' weight_str = f'''select * from tbl_sensor_collect where 1=1 and sensor_type='chickenweight' and (create_time like '{yesterday}%' or create_time like '{yesterday - timedelta(1)}%') order by create_time asc ''' image_str = f'''select * from tbl_image_analysis_weight where 1=1 and data_type = 'real' and create_time like '{yesterday}%' and weight_prediction_pixel_mean != '' order by create_time asc''' # read data from db weight_df = dbconn.select_from_db(weight_str)[['CREATE_TIME','HOUSE_ID','MODULE_ID','SENSOR_DATA']] image_df = dbconn.select_from_db(image_str)[['CREATE_TIME','HOUSE_ID','MODULE_ID','WEIGHT_PREDICTION_COUNT','WEIGHT_PREDICTION_PIXEL_MEAN']] yester_df = dbconn.select_from_db(yesterday_str) # columns to lower case weight_df.columns = [str(x).lower() for x in weight_df.columns] image_df.columns = [str(x).lower() for x in image_df.columns] yester_df.columns = [str(x).lower() for x in yester_df.columns] # setup index of yesterday date yester_df['medianweight'] = yester_df.medianweight.astype(float) yester_df = yester_df.groupby(['house_id', 'create_time'])[['medianweight']].mean() # weight df weight_df['create_time'] = pd.to_datetime(weight_df.create_time, format='%Y-%m-%d %H:%M:%S') weight_df['house_id'] = weight_df.house_id.astype(str) weight_df['module_id'] = weight_df.module_id.astype(str) weight_df['sensor_data'] = weight_df.sensor_data.astype(float) # H04만 일단 # weight_df = weight_df[weight_df.house_id == 'H04'] weight_df['gp_std'] = weight_df.groupby(['create_time', 'house_id', 'module_id'])[['sensor_data']].transform( lambda x: np.std(x)) weight_df['gp_mean'] = weight_df.groupby(['create_time', 'house_id', 'module_id'])[['sensor_data']].transform( lambda x: np.median(x)) weight_df['gp_delta'] = weight_df.groupby(['create_time', 'house_id', 'module_id'])[['sensor_data']].transform( lambda x: np.max(x) - np.min(x)) weight_gp = weight_df.groupby([pd.Grouper(key='create_time', freq='2min'), 'house_id', 'module_id']).agg( {'gp_mean': (lambda x: np.median(x))}) weight_gp.reset_index(inplace=True) weight_gp.set_index(['house_id', 'module_id'], inplace=True) weight_gp.sort_values('create_time', inplace=True) weight_gp['delta_gp'] = weight_gp.groupby(['house_id', 'module_id'])['gp_mean'].transform(fn_roll_transform).fillna( 0) weight_gp.reset_index(inplace=True) weight_gp.sort_values(['house_id', 'module_id', 'create_time'], inplace=True) weight_shifted = weight_gp weight_shifted['dateYMD'] = weight_shifted.create_time.dt.date weight_shifted['yesterdayweight'] = weight_shifted.groupby(['dateYMD', 'house_id'])['delta_gp'].transform( lambda x: np.median([y for y in x if y > isUp])) ## zero weight_shifted['zero'] = weight_shifted.groupby(['create_time', 'house_id'])['gp_mean'].transform( lambda x: np.min(x)) weight_shifted['gp_mean'] = [(x[0] - x[1]) for x in zip(weight_shifted.gp_mean, weight_shifted.zero)] # df_yesterday weight_shifted = weight_shifted[weight_shifted.dateYMD.astype(str) >= str(yesterday)] weight_shifted['yesterDate'] = (weight_shifted['dateYMD'] - timedelta(days=1)).astype(str) weight_shifted['shifted'] = [yester_df.loc[x, 'medianweight'] for x in zip(weight_shifted.house_id, weight_shifted.yesterDate)] weight_shifted['shifted'] = weight_shifted.shifted.astype(float) weight_shifted['noChickRaw'] = weight_shifted.gp_mean / weight_shifted.shifted weight_shifted['noChickRaw'] = weight_shifted.noChickRaw.fillna(0) weight_shifted['noChick'] = [(roundUp(x) if ((x > 0) & (x < 1000)) else 0) for x in weight_shifted.noChickRaw] # 올라간 닭 숫자 0 초과 1000 미만 weight_shifted['meanWeight'] = weight_shifted.gp_mean / weight_shifted.noChick weight_shifted['checkWeight'] = weight_shifted.meanWeight - weight_shifted.yesterdayweight weight_final_df = \ weight_shifted[(weight_shifted.delta_gp >= weight_shifted.yesterdayweight) & (weight_shifted.noChick > 0) & ( weight_shifted.checkWeight >= 0)]. \ groupby(['house_id', pd.Grouper(key='create_time', freq='2min')])[['noChick', 'meanWeight']].agg( {'noChick': ['count', 'mean'], 'meanWeight': 'mean'}) weight_final_df.reset_index(inplace=True) weight_final_df.columns = ['house_id', 'create_time', 'noChickCount', 'noChickMean', 'meanWeight'] # image df image_df.dropna(inplace=True) image_df['create_time'] = pd.to_datetime(image_df.create_time, format='%Y-%m-%d %H:%M:%S') image_df['house_id'] = image_df.house_id.astype(str) image_df['module_id'] = image_df.module_id.astype(str) image_df['weight_prediction_count'] = image_df.weight_prediction_count.astype(int) image_df['weight_prediction_pixel_mean'] = [(eval(x) if str(x).startswith('[') else x) for x in image_df.weight_prediction_pixel_mean] # sort weight_df, image_idf image_df.sort_values(['create_time', 'house_id', 'module_id'], inplace=True) weight_final_df.sort_values(['create_time', 'house_id'], inplace=True) # join-merge-asof # joined joined = pd.merge_asof(image_df, weight_final_df, on=['create_time'], by=['house_id'], direction="nearest", tolerance=pd.Timedelta("9min")).dropna() joined.columns = ['create_time', 'house_id', 'module_id', 'predCount', 'predPixelMean', 'noChickCount', 'noChickMean', 'meanWeight'] joined_gp = joined.groupby(['house_id', 'module_id', pd.Grouper(key='create_time', freq='1d')])[ ['predCount', 'predPixelMean', 'noChickCount', 'noChickMean', 'meanWeight']].apply(apply_fn1) joined_gp['predPixelMean'] = joined_gp.predPixelMean.apply(lambda x: sorted(list(chain(*x)), reverse=True)) joined_gp['pixelMean'] = joined_gp.predPixelMean.apply(lambda x: round(stats.trim_mean(x, 0.25), 1)) joined_gp['weightMean'] = joined_gp.meanWeight.apply( lambda x: round(stats.trim_mean([float(y) for y in x.split(',')], 0.25), 1)) joined_gp['weightMeanStd'] = joined_gp.meanWeight.apply( lambda x: round(np.std([float(y) for y in x.split(',')]) * 2, 1)) joined_gp['weightMeanStd'] = [round(max(x * .1, y), 1) for (x, y) in zip(joined_gp.weightMean, joined_gp.weightMeanStd)] joined_gp['weightMedian'] = joined_gp.meanWeight.apply( lambda x: round(np.median([float(y) for y in x.split(',')]), 1)) final_df = joined_gp[['weightMedian', 'weightMeanStd', 'pixelMean']].reset_index()[ ['create_time', 'house_id', 'module_id', 'weightMedian', 'weightMeanStd', 'pixelMean']] final_df = final_df[final_df.create_time >= str(yesterday)] final_df['create_time'] = final_df.create_time.astype(str) print(final_df) # DB insert if insertDB > 0 : try : dbConn = PyDBconnector() print('insert results===========================') for pdRow in final_df.iterrows() : insert_string = f"insert into tbl_weight_stats(create_time, house_id, module_id, medianWeight, deltaWeight, medianPixel) values('{pdRow[1]['create_time']}','{pdRow[1]['house_id']}','{pdRow[1]['module_id']}',{pdRow[1]['weightMedian']},{pdRow[1]['weightMeanStd']},{pdRow[1]['pixelMean']})" dbConn.insert_to_db(insert_string) dbConn.close() except : raise Exception('insert query error! check DB')