Tryag File Manager

//paip/script/weight/dailyDBupdator_backup.py

import os, sys
from datetime import date, timedelta
import pandas as pd
import numpy as np
import datetime
from scipy import stats

# set sys path to import PyDBconnector
HOME_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir)
DATA_DIR = 'data'
MODULE_DIR = 'util'
OUTPUT_DIR = 'out'
isUp = 50.
insertDB = 1
sys.path.append(os.path.join(HOME_PATH, MODULE_DIR))
from PyDBconnector import PyDBconnector
from utils import roundUp
from itertools import chain
import warnings
warnings.filterwarnings('ignore')

def apply_fn1(x):
    tempVal = list()
    # print(f"{type(x['predPixelMean'])}, value : {x['predPixelMean'].values}")
    tempVal.append(x['predPixelMean'].values.tolist())
    return pd.DataFrame({'predCount': [sum(x['predCount'].values)], \
                         'predPixelMean': tempVal, \
                         'noChickCount': [sum(x['noChickCount'].values)], \
                         'noChickMean': [sum(x['noChickMean'].values)], \
                         'meanWeight': ','.join([str(round(y, 1)) for y in x['meanWeight']]) \
                         })

def fn_roll_transform(x):
    df = pd.DataFrame(x)
    return df.iloc[:, 0].rolling(2).apply(lambda x: x.iloc[1] - x.iloc[0])

def fn_shift_transform(x):
    df = pd.DataFrame(x)
    return df.iloc[:, 0].shift(1)

if __name__ == '__main__':
    dbconn = PyDBconnector()

yesterday = date.today() - timedelta(1)
    # yesterday='2022-01-08'

yesterday_str = f'''select * 
                        from tbl_weight_stats
                        '''
    weight_str = f'''select *
                    from tbl_sensor_collect
                    where 1=1
                    and sensor_type='chickenweight'
                    and (create_time like '{yesterday}%' or create_time like '{yesterday - timedelta(1)}%')
                    order by create_time asc
                    '''
    image_str = f'''select 
                    *
                    from tbl_image_analysis_weight
                    where 1=1
                    and data_type = 'real'
                    and create_time like '{yesterday}%'
                    and weight_prediction_pixel_mean != ''
                    order by create_time asc'''

# read data from db
    weight_df = dbconn.select_from_db(weight_str)[['CREATE_TIME','HOUSE_ID','MODULE_ID','SENSOR_DATA']]
    image_df = dbconn.select_from_db(image_str)[['CREATE_TIME','HOUSE_ID','MODULE_ID','WEIGHT_PREDICTION_COUNT','WEIGHT_PREDICTION_PIXEL_MEAN']]
    yester_df = dbconn.select_from_db(yesterday_str)

# columns to lower case
    weight_df.columns = [str(x).lower() for x in weight_df.columns]
    image_df.columns = [str(x).lower() for x in image_df.columns]
    yester_df.columns = [str(x).lower() for x in yester_df.columns]

# setup index of yesterday date
    yester_df['medianweight'] = yester_df.medianweight.astype(float)
    yester_df = yester_df.groupby(['house_id', 'create_time'])[['medianweight']].mean()

# weight df
    weight_df['create_time'] = pd.to_datetime(weight_df.create_time, format='%Y-%m-%d %H:%M:%S')
    weight_df['house_id'] = weight_df.house_id.astype(str)
    weight_df['module_id'] = weight_df.module_id.astype(str)
    weight_df['sensor_data'] = weight_df.sensor_data.astype(float)

# H04만 일단
    # weight_df = weight_df[weight_df.house_id == 'H04']
    weight_df['gp_std'] = weight_df.groupby(['create_time', 'house_id', 'module_id'])[['sensor_data']].transform(
        lambda x: np.std(x))
    weight_df['gp_mean'] = weight_df.groupby(['create_time', 'house_id', 'module_id'])[['sensor_data']].transform(
        lambda x: np.median(x))
    weight_df['gp_delta'] = weight_df.groupby(['create_time', 'house_id', 'module_id'])[['sensor_data']].transform(
        lambda x: np.max(x) - np.min(x))

weight_gp = weight_df.groupby([pd.Grouper(key='create_time', freq='2min'), 'house_id', 'module_id']).agg(
        {'gp_mean': (lambda x: np.median(x))})
    weight_gp.reset_index(inplace=True)

weight_gp.set_index(['house_id', 'module_id'], inplace=True)
    weight_gp.sort_values('create_time', inplace=True)
    weight_gp['delta_gp'] = weight_gp.groupby(['house_id', 'module_id'])['gp_mean'].transform(fn_roll_transform).fillna(
        0)
    weight_gp.reset_index(inplace=True)
    weight_gp.sort_values(['house_id', 'module_id', 'create_time'], inplace=True)

weight_shifted = weight_gp
    weight_shifted['dateYMD'] = weight_shifted.create_time.dt.date
    weight_shifted['yesterdayweight'] = weight_shifted.groupby(['dateYMD', 'house_id'])['delta_gp'].transform(
        lambda x: np.median([y for y in x if y > isUp]))

## zero
    weight_shifted['zero'] = weight_shifted.groupby(['create_time', 'house_id'])['gp_mean'].transform(
        lambda x: np.min(x))
    weight_shifted['gp_mean'] = [(x[0] - x[1]) for x in zip(weight_shifted.gp_mean, weight_shifted.zero)]

# df_yesterday
    weight_shifted = weight_shifted[weight_shifted.dateYMD.astype(str) >= str(yesterday)]
    weight_shifted['yesterDate'] = (weight_shifted['dateYMD'] - timedelta(days=1)).astype(str)
    weight_shifted['shifted'] = [yester_df.loc[x, 'medianweight'] for x in
                                 zip(weight_shifted.house_id, weight_shifted.yesterDate)]
    weight_shifted['shifted'] = weight_shifted.shifted.astype(float)
    weight_shifted['noChickRaw'] = weight_shifted.gp_mean / weight_shifted.shifted
    weight_shifted['noChickRaw'] = weight_shifted.noChickRaw.fillna(0)
    weight_shifted['noChick'] = [(roundUp(x) if ((x > 0) & (x < 1000)) else 0) for x in
                                 weight_shifted.noChickRaw]  # 올라간 닭 숫자 0 초과 1000 미만
    weight_shifted['meanWeight'] = weight_shifted.gp_mean / weight_shifted.noChick
    weight_shifted['checkWeight'] = weight_shifted.meanWeight - weight_shifted.yesterdayweight
    weight_final_df = \
    weight_shifted[(weight_shifted.delta_gp >= weight_shifted.yesterdayweight) & (weight_shifted.noChick > 0) & (
            weight_shifted.checkWeight >= 0)]. \
        groupby(['house_id', pd.Grouper(key='create_time', freq='2min')])[['noChick', 'meanWeight']].agg(
        {'noChick': ['count', 'mean'], 'meanWeight': 'mean'})

weight_final_df.reset_index(inplace=True)
    weight_final_df.columns = ['house_id', 'create_time', 'noChickCount', 'noChickMean', 'meanWeight']

# image df
    image_df.dropna(inplace=True)
    image_df['create_time'] = pd.to_datetime(image_df.create_time, format='%Y-%m-%d %H:%M:%S')
    image_df['house_id'] = image_df.house_id.astype(str)
    image_df['module_id'] = image_df.module_id.astype(str)
    image_df['weight_prediction_count'] = image_df.weight_prediction_count.astype(int)
    image_df['weight_prediction_pixel_mean'] = [(eval(x) if str(x).startswith('[') else x) for x in
                                                image_df.weight_prediction_pixel_mean]

# sort weight_df, image_idf
    image_df.sort_values(['create_time', 'house_id', 'module_id'], inplace=True)
    weight_final_df.sort_values(['create_time', 'house_id'], inplace=True)

# join-merge-asof
    # joined
    joined = pd.merge_asof(image_df, weight_final_df, on=['create_time'], by=['house_id'], direction="nearest",
                           tolerance=pd.Timedelta("9min")).dropna()

joined.columns = ['create_time', 'house_id', 'module_id', 'predCount', 'predPixelMean', 'noChickCount',
                      'noChickMean', 'meanWeight']
    joined_gp = joined.groupby(['house_id', 'module_id', pd.Grouper(key='create_time', freq='1d')])[
        ['predCount', 'predPixelMean', 'noChickCount', 'noChickMean', 'meanWeight']].apply(apply_fn1)

joined_gp['predPixelMean'] = joined_gp.predPixelMean.apply(lambda x: sorted(list(chain(*x)), reverse=True))

joined_gp['pixelMean'] = joined_gp.predPixelMean.apply(lambda x: round(stats.trim_mean(x, 0.25), 1))
    joined_gp['weightMean'] = joined_gp.meanWeight.apply(
        lambda x: round(stats.trim_mean([float(y) for y in x.split(',')], 0.25), 1))
    joined_gp['weightMeanStd'] = joined_gp.meanWeight.apply(
        lambda x: round(np.std([float(y) for y in x.split(',')]) * 2, 1))
    joined_gp['weightMeanStd'] = [round(max(x * .1, y), 1) for (x, y) in
                                  zip(joined_gp.weightMean, joined_gp.weightMeanStd)]
    joined_gp['weightMedian'] = joined_gp.meanWeight.apply(
        lambda x: round(np.median([float(y) for y in x.split(',')]), 1))

final_df = joined_gp[['weightMedian', 'weightMeanStd', 'pixelMean']].reset_index()[
        ['create_time', 'house_id', 'module_id', 'weightMedian', 'weightMeanStd', 'pixelMean']]
    final_df = final_df[final_df.create_time >= str(yesterday)]
    final_df['create_time'] = final_df.create_time.astype(str)
    print(final_df)

# DB insert
    if insertDB > 0 :
        try :
            dbConn = PyDBconnector()
            print('insert results===========================')

for pdRow in final_df.iterrows() :
                insert_string = f"insert into tbl_weight_stats(create_time, house_id, module_id, medianWeight, deltaWeight, medianPixel) values('{pdRow[1]['create_time']}','{pdRow[1]['house_id']}','{pdRow[1]['module_id']}',{pdRow[1]['weightMedian']},{pdRow[1]['weightMeanStd']},{pdRow[1]['pixelMean']})"
                dbConn.insert_to_db(insert_string)

dbConn.close()
        except :
            raise Exception('insert query error! check DB')