Tryag File Manager

//paip/script/util/farmLiveChecker.py

import getopt, os, socket, sys, warnings, pymsteams, requests, json
from datetime import date, datetime, timedelta

import numpy as np
import pandas as pd
from logs import paiplog

warnings.filterwarnings('ignore')
HOME_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir)
MODULE_DIR = 'util'
sys.path.append(os.path.join(HOME_PATH, MODULE_DIR))

from PyDBconnector import PyDBconnector
from PaipConfigParser import PaipHostConfig

################ parameter info ################################################
# 입추 저장 테이블 정보 
TARGET_TABLE = 'tbl_house_breed_hist_analysis'

# alert Y/N
IS_ALERT = True 
# 테이블 insert, update 여부
ISINSERT, ISUPDATE= True, True #신규 row 추가 여부, out-date update 여부
# 시간 당 count 되는 닭 숫자 threshold
PARAM_NUMOFCHICK = 20
# 출하 시간 체크를 위해 참조하는 과거 시간 
CHECK_INTERVAL_CHULHA = 24

## 입추,출하 감지 시 alert 용 rest API 관련 변수
API_HOST_URL = 'http://127.0.0.1:8000/breedhist/detect_result'
API_STATUS_CODE = ['입추','출하']
################ parameter info ################################################

def checkAPIPort(host="127.0.0.1", port=8890) :
    try :
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as a_socket:
            location = (host, port)
            # 0 for opened
            result_of_check = a_socket.connect_ex(location)
            return result_of_check
    except :
        return 999

def getDBData(dbConn, sql_str) :
    try :
        return dbConn.select_from_db(sql_str)
    except :
        return None

@paiplog
def insertDatabase(dbConn, data_df, target_table=TARGET_TABLE, isInsert=True) :
    try:
        # print('insert called ===========================')
        data_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        if isInsert : # insert new InDate
            for pdRow in data_df.iterrows():
                # print(f"insertDate : {pdRow[1]['updateDate']}, type : {type(pdRow[1]['updateDate'])}")
                insert_string = f"insert into {target_table}(create_time, house_id, in_date, days_after_birth, poultry_type, kind) " \
                                f"values('{data_time}','{pdRow[1]['house_id']}','{pdRow[1]['updateDate']}',{1}, '{pdRow[1]['poultry_type']}','{pdRow[1]['kind']}')"
                dbConn.insert_to_db(insert_string)
            return True
        else : # update to modify OutDate
            pass
    except:
        raise Exception('insert query error! check DB')

@paiplog
def updateOutDate(dbConn, data_df, target_table=TARGET_TABLE, isInsert=True) :
    try:
        # print('update called ===========================')
        if isInsert : # update new outData
            for pdRow in data_df.iterrows():
                # UPDATE table_1 SET column_1 = 'z' where in_date = '';
                # print(f"out_date : {pdRow[1]['out_date']}, isnull : {pd.isnull(pdRow[1]['out_date'])}")
                if pd.isnull(pdRow[1]['out_date']) : 
                    continue 
                else :
                    updateString = f'''update {target_table} set out_date = '{pdRow[1]['out_date']}' where in_date = '{pdRow[1]['in_date']}' and house_id = '{pdRow[1]['house_id']}'
                                    '''
                    dbConn.update_to_db(updateString)
            return True
        else : # update to modify OutDate
            pass
    except:
        raise Exception('insert query error! check DB')

def checkFirst(dbConn) :
    '''
    weight count > 0 인 첫째날 데이터를 가져오는 내용
    '''
    sql_first = f'''select min(dates) as times, house_id, counts 
                    from (
                    select dates, house_id, counts, ifnull(lag(counts) over (partition by house_id order by dates),-1) as yester_counts
                    from (
                    select  date(create_time) as dates, house_id, sum(ifnull(`WEIGHT_PREDICTION_COUNT`,0)) as counts
                    from tbl_image_analysis_weight
                    group by date(create_time), house_id) a) b
                    where yester_counts < {param_numOfChick} and counts > {param_numOfChick}
                    group by house_id
                    ;'''
    breed_hist = getDBData(dbConn, sql_first)
    breed_hist['daysDelta'] = [param_dayAge+1] * len(breed_hist)
    breed_hist['out_date'] = [None] * len(breed_hist)
    breed_hist['updateDate'] = breed_hist.times

# print('checkFirst \n', breed_hist)
    return breed_hist

def getPixelCounts(dbConn, breed_hist, isIn=True) :
    if isIn : # 입추체크 
    # WHERE    create_time BETWEEN {breed_hist.in_date[0].strftime('%Y-%m-%d')} AND NOW()
        sql_str_pixel = f'''select DATE_FORMAT(create_time,'%Y-%m-%d %H') AS times, house_id, 
                    sum(ifnull(`WEIGHT_PREDICTION_COUNT`,0)) as counts
                    FROM tbl_image_analysis_weight
                    WHERE create_time BETWEEN {breed_hist.in_date[0].strftime('%Y-%m-%d')} AND NOW()
                    GROUP BY house_id, DATE_FORMAT(create_time,'%Y-%m-%d %H');'''
    else : # 출하 체크 - 현재 시각 부터 CHECK_INTERVAL_CHULHA 시간 이전 시간 내에 체크 
        sql_str_pixel = f'''select DATE_FORMAT(create_time,'%Y-%m-%d %H') AS times, house_id, 
                    sum(ifnull(`WEIGHT_PREDICTION_COUNT`,0)) as counts
                    FROM tbl_image_analysis_weight
                    WHERE create_time BETWEEN '{(pd.Timestamp.now() - pd.Timedelta(CHECK_INTERVAL_CHULHA, unit='hour')).strftime("%Y-%m-%d %H:00:00")}' AND NOW()
                    GROUP BY house_id, DATE_FORMAT(create_time,'%Y-%m-%d %H');'''
    detectionCount = getDBData(dbConn, sql_str_pixel)
    detectionCount['times'] = pd.to_datetime(detectionCount.times, format='%Y-%m-%d %H')
    # 빈 구간 채워 넣기
    detectionCount = detectionCount.groupby('house_id').apply(lambda x : x.set_index('times').resample('60min').max().fillna(0))[['counts']].reset_index()
    
    # 이전, 2칸 이전 값 groupby shift
    detectionCount['yesterdayCounts'] = detectionCount.groupby(['house_id'])['counts'].shift(fill_value=0.)
    detectionCount['thedayb4yesterdayCounts'] = detectionCount.groupby(['house_id'])['counts'].shift(2,fill_value=0.)

return detectionCount

@paiplog
def checkIsOut(dbConn, breed_hist) :
    '''
    Out_date update logic : 
    '''
    detectionCount = getPixelCounts(dbConn, breed_hist, isIn=False)
    # print(detectionCount[detectionCount.counts > 0.])
    # 출하 날짜가 null 인 경우
    for a_house_id in breed_hist.house_id : 
        upDateFlag = False
        if pd.isnull(breed_hist[breed_hist.house_id == a_house_id].out_date.values[0]) : 
            todayDayAge = [abs(np.datetime64(date.today()) - x).days for x in breed_hist[breed_hist.house_id == a_house_id].in_date][0]
            
            # 육성 중인 기간
            if todayDayAge < param_dayAge : 
                ## 출하 감지 시작일 지정
                ## 육계, 삼계 == (28.7, 21.7 ==> 26.0 19.5)
                if todayDayAge >= param_dayAge * .65 and \
                    len(detectionCount[(detectionCount.house_id == a_house_id) 
                        & (detectionCount.thedayb4yesterdayCounts > param_numOfChick) 
                        & (detectionCount.yesterdayCounts < param_numOfChick) 
                        & (detectionCount.counts == 0.)]
                    ) > 0 : 
                    breed_hist.loc[breed_hist.house_id == a_house_id,'out_date'] = pd.to_datetime(detectionCount[(detectionCount.house_id == a_house_id) & 
                                                                                                    (detectionCount.thedayb4yesterdayCounts > param_numOfChick) & 
                                                                                                    (detectionCount.yesterdayCounts < param_numOfChick) & 
                                                                                                    (detectionCount.counts == 0)]['times']).max()
                    upDateFlag = True
            # 육성 완료 이후 
            else : 
                breed_hist.loc[breed_hist.house_id == a_house_id,'out_date'] = breed_hist[breed_hist.house_id == a_house_id].in_date + timedelta(param_dayAge)
                upDateFlag = True

# update db 
        if upDateFlag : 
            # print("update db data")
            updateOutDate(dbConn, breed_hist[breed_hist.house_id == a_house_id], isInsert= ISUPDATE)
            if IS_ALERT : 
                sendAlertToRestAPI(breed_hist[breed_hist.house_id == a_house_id], isIn=False)
                # alertToTeams(breed_hist[breed_hist.house_id == a_house_id], isIn=False)
    return breed_hist

@paiplog
def getNewInDate(dbConn, breed_hist) :

detectionCount = getPixelCounts(dbConn, breed_hist)

breed_hist['updateDate'] = [None] * len(breed_hist)
    ## in_date : threshold
    for a_house_id in breed_hist.house_id : 
        # 현 시각 >  param_numOfChick, 한 시간 전 > param_numOfChick, 두 시간 전 == 0
        new_in_date_candidate = pd.to_datetime(detectionCount[(detectionCount.house_id == a_house_id) 
                                                            & (detectionCount.thedayb4yesterdayCounts < param_numOfChick) 
                                                            & (detectionCount.yesterdayCounts > param_numOfChick) 
                                                            & (detectionCount.counts > param_numOfChick)]['times']).min()
        if not pd.isnull(new_in_date_candidate) and  [abs(new_in_date_candidate - breed_hist.in_date[0]).days for x in breed_hist.in_date][0] >= param_dayAge :
            # change updateDate, daysDelta
            breed_hist.loc[breed_hist.house_id == a_house_id, 'updateDate'] = new_in_date_candidate.strftime('%Y-%m-%d %H:%M:%S')
            breed_hist.loc[breed_hist.house_id == a_house_id, 'daysDelta'] = [abs(new_in_date_candidate  - breed_hist.in_date[0]).days for x in breed_hist.in_date][0]
    if len(breed_hist[(~breed_hist.updateDate.isnull()) & (breed_hist.daysDelta > param_dayAge)]) > 0 :
        # DB Update
        insertDatabase(dbConn, breed_hist[(~breed_hist.updateDate.isnull()) & (breed_hist.daysDelta > param_dayAge)], isInsert=ISINSERT)
        #1 teams
        if IS_ALERT : 
            sendAlertToRestAPI(breed_hist[(~breed_hist.updateDate.isnull()) & (breed_hist.daysDelta > param_dayAge)], method='POST', isIn=True)
            # alertToTeams(breed_hist[(~breed_hist.updateDate.isnull()) & (breed_hist.daysDelta > param_dayAge)], isIn=True)
    return breed_hist

def alertToTeams(result_df, isIn=True) : 
    config = PaipHostConfig()
    if isIn : 
        myTeamsMessage = pymsteams.connectorcard("https://paiptree.webhook.office.com/webhookb2/971976e1-0b86-45fa-b0c8-64b9416f7b26@fe2870e7-ec9b-4920-bae0-e916e653b10c/IncomingWebhook/b13807b00728424b888fb078614eac46/85fcf23c-17d4-49f7-bff0-a8f3265d2d29")
        myTeamsMessage.text(f"Farm In_Date : {config.farmName}농장 {','.join(result_df['house_id'].unique())} 계사 입추 감지")
        print(f"Farm In_Date : {config.farmName}농장 {','.join(result_df['house_id'].unique())} 계사 입추 감지")
        myTeamsMessage.send()
    else : 
        myTeamsMessage = pymsteams.connectorcard("https://paiptree.webhook.office.com/webhookb2/971976e1-0b86-45fa-b0c8-64b9416f7b26@fe2870e7-ec9b-4920-bae0-e916e653b10c/IncomingWebhook/b13807b00728424b888fb078614eac46/85fcf23c-17d4-49f7-bff0-a8f3265d2d29")
        myTeamsMessage.text(f"Farm Out_Date : {config.farmName}농장 {','.join(result_df['house_id'].unique())} 계사 출하 감지")
        print(f"Farm Out_Date : {config.farmName}농장 {','.join(result_df['house_id'].unique())} 계사 출하 감지")
        myTeamsMessage.send()

@paiplog
def sendAlertToRestAPI(result_df, method='POST', isIn=True) : 
    # 입추 출하 선택 : 0 입추, 1 출하
    statusStr = API_STATUS_CODE[0] if isIn else API_STATUS_CODE[1] 
    headers = {'Content-Type' : 'application/json', 'charset' : 'UTF-8', 'Accept' : '*/*'}
    # param setting
    body = {
        "house_id":','.join(sorted(result_df['house_id'].unique())), 
        "status": statusStr
    }

# print(f"sendAlertToRestAPI : {json.dumps(body, ensure_ascii=False).encode('utf8')}")

try:
        if method == 'GET':
            response = requests.get(API_HOST_URL, headers=headers)
        elif method == 'POST':
            response = requests.post(API_HOST_URL, headers=headers, 
                                        data=json.dumps(body, ensure_ascii=False, indent="\t").encode('utf8') # utf8 encoding
                                    )
        print("response status %r" % response.status_code)
        print("response text %r" % response.text)
    except Exception as ex:
        raise Exception(ex)

def main(argv):

FILE_NAME     = argv[0] # command line arguments의 첫번째는 파일명
    # set up parameters
    hostIP = '127.0.0.1' # default : 127.0.0.1, gate2 : 192.100.0.11
    target_table = TARGET_TABLE
    param_dayAge = 41
    isDayAge = False
    param_numOfChick = PARAM_NUMOFCHICK

try:
        # opts: getopt 옵션에 따라 파싱 ex) [('-i', 'myinstancce1')]
        # etc_args: getopt 옵션 이외에 입력된 일반 Argument
        # argv 첫번째(index:0)는 파일명, 두번째(index:1)부터 Arguments
        opts, etc_args = getopt.getopt(argv[1:], \
                                 "hi:t:d:n:", ["help","hostIP=","target_table=","param_dayAge=","param_numOfChick="])

except getopt.GetoptError: # 옵션지정이 올바르지 않은 경우
        print(FILE_NAME, '-i <hostIP> -t <target_table> -d <param_dayAge> -n <param_numOfChick>')
        sys.exit(2)

for opt, arg in opts: # 옵션이 파싱된 경우
        if opt in ("-h", "--help"): # HELP 요청인 경우 사용법 출력
            print(FILE_NAME, '-i <hostIP> -t <target_table> -d <param_dayAge> -n <param_numOfChick>')
            sys.exit()

elif opt in ("-i", "--hostIP"):
            hostIP = arg

elif opt in ("-t", "--target_table"):
            target_table = arg

elif opt in ("-d", "--param_dayAge"):
            param_dayAge = arg
            isDayAge = True
        elif opt in ("-n", "--param_numOfChick"):
            param_numOfChick = arg

# if len(INSTANCE_NAME) < 1: # 필수항목 값이 비어있다면
    #     print(FILE_NAME, "-i option is mandatory") # 필수임을 출력
    #     sys.exit(2)
    return (hostIP,target_table,param_dayAge,isDayAge, param_numOfChick)

if __name__ == '__main__':
    # 0. setup
    type_dict = {'육계': 35, '삼계': 25, '산란계': -10}
    hostIP,target_table,param_dayAge,isDayAge, param_numOfChick = main(sys.argv)

dbConn = PyDBconnector(host=hostIP)

# 1. check chicken_weight_api.py
    ret = checkAPIPort(host=hostIP)

if ret > 0 :
        raise Exception("!!!!!!!!!!!!!!!!!!!!!!!!!!!chicken_rest_api is not running")
        # sys.exit(0)

# 2. get breed hist data - 최근 입력된 입추 정보 가져오기
    sql_str = f'''select a.*
                from {target_table} a, (select house_id, max(in_date) as max_dt from {target_table} group by house_id) b
                where a.house_id = b.house_id
                and a.in_date = b.max_dt;'''

breed_hist = getDBData(dbConn, sql_str)

if not isDayAge :
        type_sql = f'''select parameter  
                    from tbl_config 
                    where config_item = "POULTRY_TYPE"'''

type_result = getDBData(dbConn, type_sql)
        type_str = type_result.iloc[0, 0]
        param_dayAge = type_dict.get(type_str) + 5

if param_dayAge < 0:
        print('!!!!!!!!! 산란계')
        sys.exit(0)

# 3. main loop
    ## case #1 first
    # 첫 번째 case 인 경우에 자료 입력하고, 종료
    if (breed_hist is None ) or (len(breed_hist) <= 0 ) :
        print("!!!!!!!!!!!!!!!!!!!!!!!!!!!breed hist is empty")
        # param_dayAge = -1
        breed_hist = checkFirst(dbConn)
        # print('case #1 first results')
        # print(breed_hist)

insertDatabase(dbConn, breed_hist[(~breed_hist.updateDate.isnull()) & (breed_hist.daysDelta > param_dayAge)],
                             isInsert=True)
        sys.exit(0)

## case #2 insert in_date
    # column setting
    # lower column name
    breed_hist.columns = [str(x).lower() for x in breed_hist.columns]

# column type
    breed_hist['house_id'] = breed_hist.house_id.astype(str)
    breed_hist['in_date'] = pd.to_datetime(breed_hist.in_date, format='%Y-%m-%d')
    breed_hist['out_date'] = pd.to_datetime(breed_hist.out_date, format='%Y-%m-%d')
    breed_hist['daysDelta'] = [abs(x - np.datetime64(date.today())).days for x in breed_hist.in_date] # dayAge delta

# 3. get image detection count
    ###
    # test code : today_date = datetime.strptime('2022-12-14', "%Y-%m-%d").date()
    ###
    # today_date = datetime.strptime('2022-12-14', "%Y-%m-%d").date()
    # today_date = date.today()
    # yesterday = today_date - timedelta(1)

## case #3 update out_date
    if pd.isnull(breed_hist.out_date).sum() :
        outDateBreedHist = checkIsOut(dbConn, breed_hist)
        # print(f"outDateBreedHist \n : {outDateBreedHist}")

result_df = getNewInDate(dbConn, breed_hist)
    # print(f'result_df : \n {result_df}')