Source code :: pyTDMS

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""


================================================================
                Standalone Python TDMS reader, 
   (not using the NI libraries for they were Windows-specific)
================================================================


based on the format description on
http://zone.ni.com/devzone/cda/tut/p/id/5696



Floris van Vugt
IMMM Hannover
http://florisvanvugt.free.fr/


I am greatly indebted for insightful bug-corrections by:
ADAM REEVE
JUERGEN NEUBAUER, PH.D.

Thanks guys!


"""

import struct
import os





# Tells us whether we should really output messages about what
# we have read (clutters the output buffer though)
verbose = False


# store previous values for the raw data properties for when
# they are repeated
object_rawdata = {}


def byteToHex( byteStr ):
    """
    Convert a byte string to it's hex string representation e.g. for output.
    """
    
    # Uses list comprehension which is a fractionally faster implementation than
    # the alternative, more readable, implementation below
    #   
    #    hex = []
    #    for aChar in byteStr:
    #        hex.append( "%02X " % ord( aChar ) )
    #
    #    return ''.join( hex ).strip()        

    return ''.join( [ "%02X " % ord( x ) for x in byteStr ] ).strip()








tocProperties = {
    'kTocMetaData'         : (1L<<1),
    'kTocRawData'          : (1L<<3),
    'kTocDAQmxRawData'     : (1L<<7),
    'kTocInterleavedData'  : (1L<<5),
    'kTocBigEndian'        : (1L<<6),
    'kTocNewObjList'       : (1L<<2),
    }







tdsDataTypes = [
    'tdsTypeVoid',
    'tdsTypeI8',    
    'tdsTypeI16',
    'tdsTypeI32',    
    'tdsTypeI64',
    'tdsTypeU8',    
    'tdsTypeU16',    
    'tdsTypeU32',    
    'tdsTypeU64',
    'tdsTypeSingleFloat',    
    'tdsTypeDoubleFloat',    
    'tdsTypeExtendedFloat',    
    'tdsTypeSingleFloatWithUnit',    
    'tdsTypeDoubleFloatWithUnit',    
    'tdsTypeExtendedFloatWithUnit',
    'tdsTypeString',   
    'tdsTypeBoolean',   
    'tdsTypeTimeStamp',   
    'tdsTypeDAQmxRawData',
]


tdsDataTypesDefined = {
    0x19: 'tdsTypeSingleFloatWithUnit',    
    0x20: 'tdsTypeString',   
    0x21: 'tdsTypeBoolean',   
    0x44: 'tdsTypeTimeStamp',   
    0xFFFFFFFF:'tdsTypeDAQmxRawData',
}



tdsDataTypesTranscriptions = {
    'tdsTypeVoid'                  : '',
    'tdsTypeI8'                    : 'b',
    'tdsTypeI16'                   : 'h', # short: standard size: 2 bytes
    'tdsTypeI32'                   : 'l',
    'tdsTypeI64'                   : 'q',
    'tdsTypeU8'                    : 'B',
    'tdsTypeU16'                   : 'H', # unsigned short: 2 bytes
    'tdsTypeU32'                   : 'L',
    'tdsTypeU64'                   : 'Q',
    'tdsTypeSingleFloat'           : 'f',
    'tdsTypeDoubleFloat'           : 'd',
    'tdsTypeExtendedFloat'         : ' ', #NOT YET IMPLEMENTED
    'tdsTypeSingleFloatWithUnit'   : ' ', #NOT YET IMPLEMENTED
    'tdsTypeDoubleFloatWithUnit'   : ' ', #NOT YET IMPLEMENTED
    'tdsTypeExtendedFloatWithUnit' : ' ', #NOT YET IMPLEMENTED
    'tdsTypeString'                :' ', # SHOULD BE HANDLED SEPARATELY
    'tdsTypeBoolean'               :'b',
    'tdsTypeTimeStamp'             :' ', # SHOULD BE HANDLED SEPARATELY
    'tdsTypeDAQmxRawData'          :' ', # SHOULD BE HANDLED SEPARATELY
}





def dataTypeFrom( s ):
    """
    Find back the data type from
    raw input data.
    """
    repr = struct.unpack("<L", s)[0]
    if (repr in tdsDataTypesDefined.keys()):
        return tdsDataTypesDefined[repr]
    else:
        return tdsDataTypes[repr]



def dataTypeLength( datatype ):
    """
    How many bytes we need to read to
    read in an object of the given datatype
    """

    if (datatype in ['tdsTypeVoid']):
        return 0

    if (datatype in ['tdsTypeI8','tdsTypeU8','tdsTypeBoolean']):
        return 1

    if (datatype in ['tdsTypeI16','tdsTypeU16']):
        return 2

    if (datatype in ['tdsTypeI32','tdsTypeU32','tdsTypeSingleFloat','tdsTypeSingleFloatWithUnit']):
        return 4

    if (datatype in ['tdsTypeI64','tdsTypeU64','tdsTypeDoubleFloat','tdsTypeDoubleFloatWithUnit']):
        return 8

    if (datatype in ['tdsTypeTimeStamp']):
        return 16


    if (datatype in [
            'tdsTypeString',
            'tdsTypeExtendedFloat',    
            'tdsTypeExtendedFloatWithUnit',
            'tdsTypeDAQmxRawData',
            ]):
        return False



    


def dataTypeTranscription( datatype ):
    """
    Returns the identifier for the given datatype
    that we need to feed into struct.unpack to get
    the right thing out.
    """
    return tdsDataTypesTranscriptions[datatype]


    





def getValue( s, endianness, datatype ):
    """
    We just read s from the file,
    and now we need to unpack it.
    """

    if datatype=='tdsTypeTimeStamp':
        t = struct.unpack(endianness+"Qq", s)

        return (t[1],         # the number of seconds since the 1904 epoch
                t[0]*(2**-64) # plus the number of 2^-64 seconds
                )
        
    else:
        code = endianness+dataTypeTranscription(datatype)
        return struct.unpack(code, s)[0]


    return False






def readLeadIn( f ):
    """
    Read the lead-in of a segment
    """


    
    s = f.read(4) # read 4 bytes
    if (not s in ['TDSm']):
        print ("Error: segment does not start with TDSm, but with ",s)
        exit()

    
    s = f.read(4)
    toc = struct.unpack("<i", s)[0]

    metadata = {}    
    for prop in tocProperties.keys():
        metadata[prop] = (toc & tocProperties[prop])!=0
        
    #print ("TOC: "+toc)
    # Contents type (bit mask not yet decoded)


    # The version number
    s = f.read(4)
    version = struct.unpack("<i", s)[0]

    
    s = f.read(16)
    (next_segment_offset,raw_data_offset) = struct.unpack("<QQ", s)
    #print ("Length remaining: "+str(length_remaining))


    #print ("Length meta: "+str(length_meta))


    return (metadata,version,next_segment_offset,raw_data_offset)






def readObject( f ):
    """
    Read object in the metadata array
    """

    # Read the object path
    s = f.read(4)
    lnth = struct.unpack("<L", s)[0]
    objectpath = f.read(lnth)
    
        
    s = f.read(4)
    rawdataindex = struct.unpack("<L", s)[0]
    
    # No raw data associated
    if   (rawdataindex==0xFFFFFFFF):
        rawdata=()

    # Raw data index same as before
    elif (rawdataindex==0x00000000):
        rawdata = object_rawdata[objectpath]

    else:

        #print "==>Raw data reading",byteToHex(s),",that is,",rawdataindex,"bytes"

        #s = f.read(rawdataindex)

        # New raw data index!

        inf_length = rawdataindex
        
        # DataType
        s = f.read(4)
        rawdata_datatype = dataTypeFrom(s)
        
        # Dimension of the raw data array
        s = f.read(4)
        #print "Dimension: ",byteToHex(s)
        rawdata_dim = struct.unpack("<L", s)[0]


        
        # Number of raw data values
        s = f.read(8)
        rawdata_values = struct.unpack("<Q", s)[0]
            
        rawdata=(
            rawdata_datatype,
            rawdata_dim,
            rawdata_values
            )
        object_rawdata[objectpath] = rawdata

        #print "==>Done reading raw data:",rawdata

        
        
        
    # Read the number of properties
    s = f.read(4)
    nProp = struct.unpack("<L", s)[0]
    #print "Has",nProp,"properties"

    properties = {}
    for j in range(0,nProp):

        #print "Property",j
        # Read one property
        
        # Read the property name
        s = f.read(4)
        numb = struct.unpack("<L", s)[0]
        name = f.read(numb)
        #print name
        
        
        # Read the data type
        s = f.read(4)
        #print "Data type",s,byteToHex(s)
        datatype = dataTypeFrom(s)
        

        value = ''

        # If it's a string, read the length
        if (datatype=='tdsTypeString'):
            s = f.read(4)
            lengte = struct.unpack("<L", s)[0]
            value = f.read(lengte)
            
        else:
            nm = dataTypeLength( datatype )
            
            s = f.read(nm)
            value = getValue( s, "<", datatype )
            
            
        properties[name]=(datatype,value)



    return (objectpath,
            rawdataindex,
            rawdata,
            properties)






def mergeProperties( prop, newprop ):
    """
    Merge the two property lists, using the newprop
    list to overwrite if conflicts arise.
    """
    
    # What we will return
    retprop = prop

    # Now we change the values wherever we need to
    for k in newprop.keys():
        retprop[k]=newprop[k]

    # And the return the merged list
    return retprop





def mergeObject( obj, newobj ):
    """
    Ok, we are given two objects: obj and alt.
    We make all the changes (new values or
    overwriting old values), taking newobj as
    dominant.
    """

    (objectpath,
     rawdataindex,
     rawdata,
     properties) = obj

    (newobjectpath,
     newrawdataindex,
     newrawdata,
     newproperties) = newobj


    # We assume that objectpath is the same
    if (newobjectpath!=objectpath):
        print "Error: trying to merge non-same objectpaths:",newobjectpath,objectpath
        exit()


    # If there is some change in the raw data associated
    if (not (newrawdataindex in [0xFFFFFFFF,0x00000000])):
        retrawdataindex = newrawdataindex
        retrawdata      = newrawdata
    else:
        retrawdataindex = rawdataindex
        retrawdata      = rawdata


    return (objectpath,
            retrawdataindex,
            retrawdata,
            mergeProperties(properties,newproperties))

    



def mergeObjects( objects, newobjects ):
    """
    Return the objects (metadata), but
    add the stuff that is in newobjects.
    """
    retobjects = objects

    # For all the new objects...
    for obj in newobjects.keys():

        # See if there is an old version already
        if (obj in retobjects.keys()):

            # Then update the old version using the new information
            retobjects[obj] = mergeObject(retobjects[obj],newobjects[obj])
        else:

            # Else just add it anew
            retobjects[obj] = newobjects[obj]

    return retobjects








def readMetaData( f ):
    """
    Read meta data from file f.
    
    We return (objects,objectorder) where
    objects is the structure containing all information about
    objects, and objectorder is a list of objectpaths (object ID's if you want)
    in the order that they have been presented. We need this
    later when we start reading the raw data, since it then comes
    in this very order.
    """

    # The number of objects in this metadata
    s = f.read(4)
    nObjects = struct.unpack("<l", s)[0]

    
    objects     = {}
    objectorder = []

    for i in range(0,nObjects):

        obj = readObject(f)

        (objectpath,
         rawdataindex,
         rawdata,
         properties) = obj

        if verbose:
            print "Read object",objectpath

        # Add this object, or, if an object with the same objectpath
        # exists already, make it update that one.
        if (objectpath in objects.keys()):
            objects[objectpath] = mergeObjects(objects[objectpath],obj)
        else:
            # We add it anew
            objects[objectpath] = obj
            objectorder.append( objectpath )

        


    return (objects,objectorder)





def isChannel(obj):
    """
    Tell us whether the given object is a channel
    (in the current segment) and if so, returns
    the meta information about the raw data.
    """
    (_,rawdataindex,_,_) = obj
    return rawdataindex!=0xFFFFFFFF









def readRawData( f, leadin, segmentobjects, objectorder, filesize ):
    """
    Read raw data from file f,
    given the previously read leadin.
    segmentobjects are the objects that are given in this segment.
    Objectorder is a list of objectpaths (object id's) that shows
    the order in which the objects are given in the metadata. 
    That is important, for that will be the order in which their
    raw data needs to be read.
    """

    (metadata,version,next_segment_offset,raw_data_offset) = leadin

    # Whether the channel data is interleaved
    interleaved = metadata["kTocInterleavedData"]

    # Set the correct endianness (still need to check this!)
    endianness = '<'
    if metadata['kTocBigEndian']: endianness = '>'



    # First see which objects are channels (or really
    # actually which objects are channels AND have data in this segment.
    channel_sizes = {}
    channels = [ obj for obj in objectorder if isChannel(segmentobjects[obj]) ]


    for c in channels:
        channel = segmentobjects[c]

        (name,rawdataindex,rawdata,values)=channel
        
        (rawdata_datatype, rawdata_dim, rawdata_values) = rawdata

        if (rawdata_dim!=1):
            print "Error! Raw data dimension is ",rawdata_dim," and should have been 1."
            exit()
        
        # Calculate how many bytes a single value is
        datapointsize= dataTypeLength(rawdata_datatype)
        
        # Array dimension (should really be 1)
        channel_size = datapointsize * rawdata_dim * rawdata_values
        
        channel_sizes[c] = channel_size
        

        
    # How much data in all channels together
    chunk_size = sum([ channel_sizes[c] for c in channels ])


    # A correction given on the TDMS specification website
    if next_segment_offset==-1:
        next_segment_offset=filesize

    # Raw data size of total chunks
    # (next_segment_offset should already have been corrected if -1)
    total_chunks = next_segment_offset - raw_data_offset
    # Hm, I think this quantity should be the total data
    # in this segment.


    n_chunks = total_chunks / chunk_size
    if (total_chunks % chunk_size) != 0:
        raise ValueError("Data size is not a multiple of the chunk size")
    
    
    if verbose:
        print "Ready for reading",total_chunks,"bytes (",chunk_size, ") in",n_chunks,"chunks",
            

            
    








    if interleaved:

        print " ==> Interleaved"

        # Initialise data to be empty
        data = {}
        for c in channel: data[c]=[]

        j=0

        while j<chunk_size:
            
            for c in channel:
                channel = segmentobjects[c]
                
                (name,
                 rawdataindex,
                 (datatype, rawdata_dim, rawdata_values),
                 values) = channel
                
                # Calculate how many bytes a single value is
                datapointsize= dataTypeLength(rawdata_datatype)

                # Read one value from the file
                s = f.read(datapointsize)
                value = getValue( s, endianness, datatype )

                data[c].append( value )

            j+=1



    else:
        if verbose:
            print " ==> Not Interleaved"
        data = {}

        # Start with no data in our channel
        for c in channels:
            data[c] = []
        for chunk in range(n_chunks):
            for c in channels:
                size= channel_sizes[c]
                (name,
                 rawdataindex,
                 (datatype, rawdata_dim, rawdata_values),
                 values) = segmentobjects[c]
                # Calculate how many bytes a single value is
                datapointsize= dataTypeLength(datatype)
                for j in range(0,rawdata_values):
                    # Read one value from the file
                    s = f.read(datapointsize)
                    value = getValue(s,endianness,datatype)
                    data[c].append( value )

    return data









def mergeRawData( rawdata, newrawdata ):
    """
    Return the raw data, appended the new
    raw data.
    """
    for channel in newrawdata.keys():

        # If we already had data on this channel
        if (channel in rawdata.keys()):
            rawdata[channel].extend(newrawdata[channel])

        # Else we just chart it annew
        else:
            rawdata[channel] = newrawdata[channel]
    return rawdata

    







def readSegment( f, filesize, data ):
    """
    Read a segment from file f, whose filesize is given,
    and data is what we have read already
    """

    # This is the data we have so far.
    # The stuff in this segment is going to append to this.
    (objects,rawdata)=data


    leadin = readLeadIn(f)
    (metadata,version,next_segment_offset,raw_data_offset) = leadin


    newobjects = {}
    # If the segment has metadata...
    if (metadata["kTocMetaData"]):

        # Read the meta data
        (newobjects,newobjectorder) = readMetaData(f)

        # Merge the new information with what we knew already about the objects.
        objects = mergeObjects( objects, newobjects )



    if (metadata["kTocRawData"]):

        # Read the raw data
        newdata = readRawData(f,leadin,newobjects,newobjectorder,filesize)

        # And merge the data we just read with what we knew already
        rawdata = mergeRawData( rawdata, newdata )
        

    return (objects,rawdata)







def dumpProperties(props):
    ret = ''
    for pr in props:
        (tp,val)=props[pr]
        ret = ret + (pr+'=') + str(val) + ", "
    return ret



def csvDump((objects,data)):
    """
    Dump the (objects,rawdata) that we read from a TDMS file
    straight into a CSV file.
    """

    ret = ''
    for obj in objects.keys():

        # Objects
        (objectpath,
         rawdataindex,
         rawdata,
         properties) = objects[obj]
        
        print ("OBJECT "+objectpath+" ("+dumpProperties(properties)+")\n")
        # ret = ret + ''

    i = 0
    maxi = max([ len(data[obj]) for obj in objects.keys() if obj in data.keys() ])

    channels = [ obj for obj in objects.keys() if isChannel(objects[obj]) ]
    ret += '\t'.join(channels)+'\n'

    for i in range(maxi):
        
        for obj in channels:

            val = ''
            if ((obj in data.keys()) and i<len(data[obj])):
                val = str(data[obj][i])
                # The raw data associated with the object
        
            ret += val+"\t"

        ret += "\n"

    return ret









def addTimeTrack( obj, channel ):
    """
    Ok, so we've read the data. Now it's possible that we require some post-processing. For example, if at least one track has time-data set, we'll add a corresponding time vector.

    So channel is the channel for which we want to have the time vector.
    And object contains its meta data.

    We return False if we can't find time data
    
    """

    # Now check for each object whether it has 
    (objectpath, rawdataindex, rawdata, properties) = obj

    if (
        # If we have time-data for this track
        'wf_increment'     in properties.keys() and
        'wf_start_offset'  in properties.keys()
        ):

        (_,incr)   = properties['wf_increment']
        (_,offset) = properties['wf_start_offset']

        

        # Then build the time track
        timetrack = [ offset+(i*incr) for i in range(0,len(channel)) ]
        return timetrack


    # Else we can't find time data
    return False
    







def read( filename ):
    """
    Reads TDMS file with the given filename.
    We return the data, which is, object meta data and raw channel data.

    Notice that we do not read the (optionally) accompanying .tdms_index
    since it is supposed to be an exact copy of the .tdms file, without the
    raw data. So it should contain nothing new.
    """



    # We start with empty data
    data = ({},{})

    


    # Then we read the data from a file, and return that
    f = open(filename, "rb")  # Open in binary mode for portability
    sz = os.path.getsize(filename)


    
    # While there's still something left to read
    while f.tell()<sz:

        #Now we read segment by segment
        data = readSegment(f,sz,data)

    f.close()




    return data
    
#from matplotlib.pyplot import *
#from numpy import *

#items = [
#"/'Untitled'/'ch0'",
#"/'Untitled'/'ch1'",
#"/'Untitled'/'ch2'",
#"/'Untitled'/'ch3'",
#"/'Untitled'/'ch4'",
#"/'Untitled'/'ch5'",
#"/'Untitled'/'ch6'",
#"/'Untitled'/'ch7'",
#"/'Untitled'/'Voltage_0'",
#"/'Untitled'/'Voltage_1'",
#"/'Untitled'/'Voltage_2'",
#"/'Untitled'/'Voltage_3'",
#"/'Untitled'/'Voltage_4'",
#"/'Untitled'/'Voltage_5'",
#"/'Untitled'/'Voltage_6'",
#"/'Untitled'/'Voltage_7'"]

#d = read( 'NIdata_6358.tdms' )[1]
#tvec = d["/'Untitled'/'Time'"]
#tvec = array(tvec)
#tvec =tvec[:,1] - tvec[0,1] 
#i = 0

#for key in items:
    ##if key == "/'Untitled'/'Time'":
	##continue
    #i+=1
    ##print i, key

    ##print shape(d[key])
    #savez_compressed('NIturbo_%2.2i' %i,single(d[key]))
    ##list_data.append(single(d[key]))
    ##plot(tvec, d[key])
    ##title(key)
    ##show()
    ##print d[key][0]
##res = array(list_data)
##print shape()

##csvDump((objects,data)):