四种将序列数据转换为Pandas DataFrame数据方法效率对比

缺乏、安全感 2021-11-27 05:56 336阅读 0赞

该测试程序是将protocol变量转换为DataFrame类型为了便于使用numpy等数学工具  
ImuEncode\_pb2 是使用protocol生成的model

这是我使用的一些变量的声明

import time 
    import ImuEncode_pb2
    import numpy as np
    import pandas as pd 
    from pandas.core.frame import DataFrame
    from ImuSignal import LegString
    from ImuSignal import LegPositionString
    from ImuSignal import SignalAttributeDict
    from OnlineClassifier import OnlineClassifier
    
    ImuSignalDataset = ImuEncode_pb2.ImuSignalDataset()
    for i in range(50):
        imusignal =ImuSignalDataset.imusignal.add()
        for j in range(4):
            signal = imusignal.signal.add()
            signal.TimeStamp = 0
            signal.acc.AccX = 0
            signal.acc.AccY = 0
            signal.acc.AccZ = 0
            signal.angle.Roll = 0
            signal.angle.Pitch = 0
            signal.anglerate.Course = 0
            signal.quaternion.v = 0
            signal.quaternion.x = 0
            signal.quaternion.y = 0
            signal.quaternion.z = 0
            signal.leg = ImuEncode_pb2.Signal.RT
            signal.legposition = ImuEncode_pb2.Signal.Foot
    
    rate = 100 #sample rate
    period = 0.5 #classify period
    Leg = ['RT','LT']
    Leg_Position = ['Foot','Thigh']
    angle = ['Roll','Pitch','Course']
    accel = ['AccX','AccY','AccZ']

特别注意下面代码段的加入，这是对于内存的声明，我是随便赋值不知道Python变量的声明原理后续完善

该方法不简单并且耗时应该是最长的，测试耗时为： 0.09793972969055176

SignalFrame = DataFrame()
    for leg in Leg:
        for leg_position in  Leg_Position:
            for signal in  accel + angle:
                SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000
    
    starttime = time.time()
    SignalFrame = pd.concat([pd.DataFrame(data={"Foot_AccX_RT":imusignal.signal[0].acc.AccX,  "Foot_AccY_RT":imusignal.signal[0].acc.AccY,        "Foot_AccZ_RT":imusignal.signal[0].acc.AccZ,\
                "Foot_Roll_RT":imusignal.signal[0].angle.Roll,  "Foot_Pitch_RT":imusignal.signal[0].angle.Pitch,    "Foot_Course_RT":imusignal.signal[0].anglerate.Course,\
                "Thigh_AccX_RT":imusignal.signal[1].acc.AccX,   "Thigh_AccY_RT":imusignal.signal[1].acc.AccY,       "Thigh_AccZ_RT":imusignal.signal[1].acc.AccZ,\
                "Thigh_Roll_RT":imusignal.signal[1].angle.Roll, "Thigh_Pitch_RT":imusignal.signal[1].angle.Pitch,   "Thigh_Course_RT":imusignal.signal[1].anglerate.Course,\
                "Foot_AccX_LT":imusignal.signal[2].acc.AccX,    "Foot_AccY_LT":imusignal.signal[2].acc.AccY,        "Foot_AccZ_LT":imusignal.signal[2].acc.AccZ,\
                "Foot_Roll_LT":imusignal.signal[2].angle.Roll,  "Foot_Pitch_LT":imusignal.signal[2].angle.Pitch,    "Foot_Course_LT":imusignal.signal[2].anglerate.Course,\
                "Thigh_AccX_LT":imusignal.signal[3].acc.AccX,   "Thigh_AccY_LT":imusignal.signal[3].acc.AccY,       "Thigh_AccZ_LT":imusignal.signal[3].acc.AccZ,\
                "Thigh_Roll_LT":imusignal.signal[3].angle.Roll, "Thigh_Pitch_LT":imusignal.signal[3].angle.Pitch,   "Thigh_Course_LT":imusignal.signal[3].anglerate.Course,\
            },index=[0]) for imusignal in ImuSignalDataset.imusignal],ignore_index=True)
    endtime = time.time()
    print("cost time:",endtime-starttime)

该方法不简单并且耗时长，测试耗时为： 0.04797029495239258

SignalFrame = DataFrame()
    
    for leg in Leg:
        for leg_position in  Leg_Position:
            for signal in  accel + angle:
                SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000
    
    starttime = time.time()
    for imusignal in ImuSignalDataset.imusignal:
        SignalFrame = SignalFrame.append({"Foot_AccX_RT":imusignal.signal[0].acc.AccX,  "Foot_AccY_RT":imusignal.signal[0].acc.AccY,        "Foot_AccZ_RT":imusignal.signal[0].acc.AccZ,\
                "Foot_Roll_RT":imusignal.signal[0].angle.Roll,  "Foot_Pitch_RT":imusignal.signal[0].angle.Pitch,    "Foot_Course_RT":imusignal.signal[0].anglerate.Course,\
                "Thigh_AccX_RT":imusignal.signal[1].acc.AccX,   "Thigh_AccY_RT":imusignal.signal[1].acc.AccY,       "Thigh_AccZ_RT":imusignal.signal[1].acc.AccZ,\
                "Thigh_Roll_RT":imusignal.signal[1].angle.Roll, "Thigh_Pitch_RT":imusignal.signal[1].angle.Pitch,   "Thigh_Course_RT":imusignal.signal[1].anglerate.Course,\
                "Foot_AccX_LT":imusignal.signal[2].acc.AccX,    "Foot_AccY_LT":imusignal.signal[2].acc.AccY,        "Foot_AccZ_LT":imusignal.signal[2].acc.AccZ,\
                "Foot_Roll_LT":imusignal.signal[2].angle.Roll,  "Foot_Pitch_LT":imusignal.signal[2].angle.Pitch,    "Foot_Course_LT":imusignal.signal[2].anglerate.Course,\
                "Thigh_AccX_LT":imusignal.signal[3].acc.AccX,   "Thigh_AccY_LT":imusignal.signal[3].acc.AccY,       "Thigh_AccZ_LT":imusignal.signal[3].acc.AccZ,\
                "Thigh_Roll_LT":imusignal.signal[3].angle.Roll, "Thigh_Pitch_LT":imusignal.signal[3].angle.Pitch,   "Thigh_Course_LT":imusignal.signal[3].anglerate.Course,\
            },ignore_index=True)
    endtime = time.time()
    print("cost time:",endtime-starttime)

该方法简单并且耗时仅长于最快的实现方法，测试耗时为：0.016989469528198242

该方法不简单，如果数据量较大可能会影响数据的准确度，但是耗时应该是最短的，测试耗时为：0.012990951538085938

SignalFrame = DataFrame()
    
    for leg in Leg:
        for leg_position in  Leg_Position:
            for signal in  accel + angle:
                SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000
    
    starttime = time.time()
    i=0
    for imusignal in ImuSignalDataset.imusignal:
        SignalFrame.loc[i]=[imusignal.signal[0].acc.AccX   , imusignal.signal[0].acc.AccY    , imusignal.signal[0].acc.AccZ,\
                                imusignal.signal[0].angle.Roll  , imusignal.signal[0].angle.Pitch , imusignal.signal[0].anglerate.Course,\
                                imusignal.signal[1].acc.AccX    , imusignal.signal[1].acc.AccY    , imusignal.signal[1].acc.AccZ,\
                                imusignal.signal[1].angle.Roll  , imusignal.signal[1].angle.Pitch , imusignal.signal[1].anglerate.Course,\
                                imusignal.signal[2].acc.AccX    , imusignal.signal[2].acc.AccY    , imusignal.signal[2].acc.AccZ,\
                                imusignal.signal[2].angle.Roll  , imusignal.signal[2].angle.Pitch , imusignal.signal[2].anglerate.Course,\
                                imusignal.signal[3].acc.AccX    , imusignal.signal[3].acc.AccY    , imusignal.signal[3].acc.AccZ,\
                                imusignal.signal[3].angle.Roll  , imusignal.signal[3].angle.Pitch , imusignal.signal[3].anglerate.Course]
        i+=1
    endtime = time.time()
    print("cost time:",endtime-starttime)

用于生成ImuEncode\_pb2的ImuEncode.proto文件如下

syntax = "proto2";
    package ImuTutorial;
    
    message Signal {
      enum Leg {
        RT = 0;
        LT = 1;
      }
      enum LegPosition{
        Foot = 0;
        Thigh = 1;
      }
    
      required Leg leg = 1 [default = RT];
      required LegPosition legposition = 2 [default = Foot];
      optional float TimeStamp = 3;
      optional int64 time = 4;
      optional int32 id = 5;
    
      message Acc{
        required float AccX = 1;
        required float AccY = 2;
        required float AccZ = 3;
      }
      message Angle{
        optional float Course = 1;
        required float Roll = 2;
        required float Pitch = 3;
      }
      message AngleRate{
        required float Course = 1;
        optional float Roll = 2;
        optional float Pitch = 3;
      }
      message Quaternion{
        optional float v = 1;
        required float x = 2;
        required float y = 3;
        required float z = 4;
      }
      required Acc acc = 6;
      required Angle angle= 7;
      required AngleRate anglerate= 8;
      optional Quaternion quaternion = 9;
    }
    
    message ImuSignal {
      required uint64 time = 1;
      repeated Signal signal = 2;  
    }
    
    message ImuSignalDataset {
      repeated ImuSignal imusignal = 1;
    }
    
    message Datalength {
      required fixed32 length = 1;
    }

转载于:https://www.cnblogs.com/FlameBlog/p/10939978.html