Python read raw data

def ReadRawSAData(DataDirectory, fname_prefix):
    """
    This function reads in the raw SA data to a pandas dataframe

    Args:
        DataDirectory: the data directory
        fname_prefix: the file name prefix

    Returns:
        pandas dataframe with the raw SA data

    Author: FJC
    """
    
    # get the csv filename
    fname_suffix = "_SAvertical.csv"
    fname = fname_prefix+fname_suffix
    df = pd.read_csv(DataDirectory+fname)

    return df 

def read_metadata_and_raw_data(self):
        """ Override AbstractConnection.read_metadata_and_raw_data()
        """
        size_bytes = await self.__read_fully(4, allow_eof=True)
        if size_bytes is None:
            return None, None
        size = int.from_bytes(size_bytes, byteorder="big")

        if self.command_size_limit is not None and size > self.command_size_limit:
            raise RuntimeError("{}: command package exceed limit".format(self.name))

        metadata_bytes = await self.__read_fully(self.metadata_class.get_byte_size())
        metadata = self.metadata_class.deserialize(metadata_bytes)

        raw_data_without_size = await self.__read_fully(1 + 8 + size)
        return metadata, raw_data_without_size 

def readRawData(self):
        self.__startMeasuring__()
        for i in range(self.VAL_RETRIES):
            #C-code example from sensor manufacturer suggest to wait 100ms (Duration of the measurement)
            # no to get the very last measurement shoudn't be a problem -> wait 10ms
            # try a read every 10 ms for maximum VAL_RETRIES times
            sleep(.01)
            data_bytes=self.readBytes(4)
            stale_bit = (data_bytes[0] & 0b01000000) >> 6
            if (stale_bit == 0):    
                raw_t = ((data_bytes[2] << 8) | data_bytes[3]) >> 2
                raw_h = ((data_bytes[0] & 0b00111111) << 8) | data_bytes[1]
                return (raw_t, raw_h)

        #Stale was never 0, so datas are not actual
        raise Exception("HYT221(slave=0x%02X): data fetch timeout" % self.slave) 

def read_raw_data(file_key, rp):
    """
    This method reads the dataset present as it is originally and
    forms the structure which will be easy to use for the further process.

    :argument:
        :param file_key: A string which represents the raw data file, in properties.conf,
                          used for the process (experiment).
        :param rp: Absolute path of the root directory of the project

    :Expects:
        Expected line format "coarse_class:fine_class This is the question string"
    :returns:
        boolean flag: True for successful operation
        coarse_classes_list: List of coarse classes for questions
        fine_classes_list: List of fine classes for questions
        questions_list: List of the questions in the raw data file
    :Example for a single line:
        ENTY:cremat What films featured the character Popeye Doyle ?
        :return: coarse_classes_list = ['ENTY'], fine_classes_list = ['cremat'],
                 questions_list = ['What films featured the character Popeye Doyle ?']
    """
    coarse_classes_list = []
    fine_classes_list = []
    questions_list = []
    flag, file = read_file(file_key, rp)
    if flag:
        for line in file:
            space_separated_row = line.split(" ")
            classes = space_separated_row[0].split(":")
            question = " ".join(space_separated_row[1:])
            coarse_class, fine_class = classes[0], classes[1]
            coarse_classes_list.append(coarse_class)
            fine_classes_list.append(fine_class)
            questions_list.append(question)
        file.close()
        return True, coarse_classes_list, fine_classes_list, questions_list
    else:
        return False 

def read_raw_data(self):
    data = dict()
    for name in self.config.data_files:
      raw = []
      tf.logging.info(
        'Reading data file {}'.format(self.config.data_files[name]))
      for line in open(self.config.data_files[name]):
        question, answers = line.strip().split('\t')
        question = question.replace('],', ']')  # ignore ',' in the template
        raw.append((question, answers))
      data[name] = raw
    return data 

def readRawData(self, data, offset):
        "read the raw data, don't process anything, just get the bytes"
        if self.type == 0:
            return

        num_bytes = self.pad(self.size * self.repeat)
        if num_bytes == 0:
            # empty package.
            rawdata = None
        else:
            fmt = '>' + str(num_bytes) + 's'
            s  = struct.Struct(fmt)
            rawdata, = s.unpack_from(data, offset=offset+8)

        return(rawdata) 

def read_raw_data(data_fn: str):
    r"""
    Reads raw data from a file. Each line contains one example.
    """
    examples = []
    with open(data_fn, "r") as fin:
        for line in fin:
            examples.append(line.strip())
    return examples 

def read_raw_data(run_index, hcp_params, lfreq=0.5, hfreq=60):
    raw = hcp.read_raw(run_index=run_index, **hcp_params)
    raw.load_data()
    # apply ref channel correction and drop ref channels
    # preproc.apply_ref_correction(raw)

    annots = hcp.read_annot(run_index=run_index, **hcp_params)
    # construct MNE annotations
    bad_seg = (annots['segments']['all']) / raw.info['sfreq']
    annotations = mne.Annotations(
        bad_seg[:, 0], (bad_seg[:, 1] - bad_seg[:, 0]),
        description='bad')

    raw.annotations = annotations
    raw.info['bads'].extend(annots['channels']['all'])
    raw.pick_types(meg=True, ref_meg=False)

    raw.filter(lfreq, None, method='iir',
               iir_params=dict(order=4, ftype='butter'), n_jobs=1)
    raw.filter(None, hfreq, method='iir',
               iir_params=dict(order=4, ftype='butter'), n_jobs=1)

    # read ICA and remove EOG ECG
    # note that the HCP ICA assumes that bad channels have already been removed
    ica_mat = hcp.read_ica(run_index=run_index, **hcp_params)

    # We will select the brain ICs only
    exclude = annots['ica']['ecg_eog_ic']
    preproc.apply_ica_hcp(raw, ica_mat=ica_mat, exclude=exclude)
    return raw 

def read_raw_data(self):
        directory_path = './data/Foursquare/'
        checkin_file = 'Foursquare_checkins.txt'
        all_data = open(directory_path + checkin_file, 'r').readlines()
        sparse_raw_matrix = sparse.dok_matrix((self.user_num, self.poi_num))
        for eachline in all_data:
            uid, lid, time = eachline.strip().split()
            uid, lid = int(uid), int(lid)
            sparse_raw_matrix[uid, lid] = sparse_raw_matrix[uid, lid] + 1

        return sparse_raw_matrix.tocsr() 

def read_raw_data(self):
        """Read number of bytes specified by a signed int.

        First a 32-bit signed integer is read, then
        that number of bytes is read from the stream.

        :returns: Tuple (bytes_read, bytes)
        """
        length = struct.unpack("<i", self.read(4))[0]
        buf = self.read(length)

        return length, buf 

def read_raw_data(data_fn):
    """
    Reads raw data from a file. Each line contains one example.
    """
    examples = []
    with open(data_fn, "r") as fin:
        for line in fin:
            examples.append(line.strip())
    return examples 

def read_data_raw(self, filename, offset, len):
        f = os.open(filename, os.O_RDONLY)
        os.lseek(f, offset, os.SEEK_SET)
        data = os.read(f, len)
        os.close(f)
        return data 

def read_raw_data(data_fn):
    """
    Reads raw data from a file. Each line contains one example.
    """
    examples = []
    with open(data_fn, 'r') as fin:
        for line in fin:
            examples.append(line.strip())
    return examples 

def read_raw_data(addr):
	#Accelero and Gyro value are 16-bit
        high = bus.read_byte_data(DeviceAddress, addr)
        low = bus.read_byte_data(DeviceAddress, addr+1)

        #concatenate higher and lower value
        value = ((high << 8) | low)

        #to get signed value from mpu6050
        if(value > 32768):
                value = value - 65536
        return value 

def read_raw_data(self, result):
        """ Reads the raw (uncompensated) data from the sensor.

            Args:
                result: array of length 3 or alike where the result will be
                stored, in temperature, pressure, humidity order
            Returns:
                None
        """

        self._l1_barray[0] = self._mode_temperature << 5 | self._mode_pressure << 2 | BMP280_POWER_MODE_FORCED
        self.i2c.writeto_mem(self.address, BMP280_REGISTER_CONTROL, self._l1_barray)

        time.sleep_us(self.compute_delay_time())  # Wait the required time

        # burst readout from 0xF7 to 0xFC, recommended by datasheet
        # we read the 6 bytes (3 bytes each)
        self.i2c.readfrom_mem_into(self.address, 0xF7, self._l8_barray)
        readout = self._l8_barray
        # pressure(0xF7): ((msb << 16) | (lsb << 8) | xlsb) >> 4
        raw_press = ((readout[0] << 16) | (readout[1] << 8) | readout[2]) >> 4
        # temperature(0xFA): ((msb << 16) | (lsb << 8) | xlsb) >> 4
        raw_temp = ((readout[3] << 16) | (readout[4] << 8) | readout[5]) >> 4

        result[0] = raw_temp
        result[1] = raw_press 

def read_data_raw(self, max_requests=50):
        with self.co2hid(send_magic_table=True):
            vals = self._read_co2_temp(max_requests=max_requests)
            self._last_data = vals
            return vals 

def read_raw_data_from_file(self, **sample_fnames) -> dict:
        """
        Abstract method to reed the raw data using :obj:`tf.data.Dataset`

        Parameters
        ----------
        sample_fnames
            sample with file names to read the data from

        Returns
        -------
        data
            data read from sample_fnames
        """ 

def read_file_raw_data(self, file_path):
    file_path = file_path.lower()
    if file_path not in self.files_by_path_lowercase:
      raise Exception("Could not find file: " + file_path)
    
    file_entry = self.files_by_path_lowercase[file_path]
    with open(self.iso_path, "rb") as iso_file:
      data = read_bytes(iso_file, file_entry.file_data_offset, file_entry.file_size)
    
    return data 

def read_raw_data_v2(path, mode):
    def _read_file(fn):
        with open(fn, 'r') as fin:
            lines = [line.strip() for line in fin]
        return lines

    def _get_fn(field):
        return os.path.join(path, '%s_%s.txt' % (mode, field))

    all_x1 = _read_file(_get_fn('x1'))
    all_x2 = _read_file(_get_fn('x2'))
    all_xx2 = _read_file(_get_fn('xx2'))
    all_y = _read_file(_get_fn('y'))

    yy_fn = _get_fn('yy')
    if os.path.isfile(yy_fn):
        all_yy = _read_file(yy_fn)
    else:
        all_yy = [None] * len(all_x1)

    print('#examples: %d' % len(all_x1))

    return [
        InputExample(
            x1=x1,
            x2=x2,
            xx2=xx2,
            y=y,
            yy=yy)
        for x1, x2, xx2, y, yy in zip(all_x1, all_x2, all_xx2, all_y, all_yy)
    ] 

def read_raw_data(data_format, data_file_name=None):
    if data_format == "23andme":
        if not data_file_name is None:
            return twenty_three_and_me(data_file_name)
        else:
            return twenty_three_and_me(
                os.path.join(
                    os.path.dirname(__file__), "data/demo_genome_23andme.txt"))
    elif data_format == 'ancestry':
        if not data_file_name is None:
            return ancestry(data_file_name)
        else:
            print("Data file not set!")
            exit()
            return None
    elif data_format == 'ftdna':
        if not data_file_name is None:
            return ftdna(data_file_name)
        else:
            print("Data file not set!")
            exit()
            return None
    elif data_format == 'ftdna2':
        if not data_file_name is None:
            return ftdna2(data_file_name)
        else:
            print("Data file not set!")
            exit()
            return None
    elif data_format == 'wegene':
        if not data_file_name is None:
            return wegene(data_file_name)
        else:
            print("Data file not set!")
            exit()
            return None
    elif data_format == 'myheritage':
        if not data_file_name is None:
            return myheritage(data_file_name)
        else:
            print("Data file not set!")
            exit()
            return None
    else:
        print("Data format does not exist!")
        exit()
        return None


# convert alleles information of a model to a dict 

def read_raw_data(datafile, dtype, shape, use_mmap=False, offset=0,
                  order='C', partial_read=False):
    """Read a raw binary file and shape it.

    PARAMETERS
    ----------
    datafile : str
        Path to a .data file
    dtype : numpy.dtype
        Data type of the data
    shape : tuple
        Shape of the data
    use_memmap : bool, optional
        Whether to read the data using a numpy.memmap
    offset : int, optional
        Offset (in bytes) to apply on read
    order : str, optional
        Row/Column Major = 'C' or 'F'
    partial_read : bool, optional
        If reading part of the file

    RETURNS
    -------
    data : numpy.ndarray
        The data (or a memmap to it)
    """

    number_of_values = reduce(lambda x, y: x * y, shape)
    expected_number_of_bytes = number_of_values * dtype.itemsize
    actual_number_of_bytes = os.path.getsize(datafile)
    if not partial_read:
        # first check that partial_read and offset are used together
        if offset != 0:
            raise ValueError(
                'When partial_read==False, offset will not be read')
        # second check to be sure there is the right number of bytes in file
        if expected_number_of_bytes != actual_number_of_bytes:
            raise IOError('File `%s` does not have the correct size '
                          '(expected %g, found %g)' %
                          (datafile,
                           expected_number_of_bytes,
                           actual_number_of_bytes))
    else:
        pass

    if offset < actual_number_of_bytes:
        pass
    else:
        raise ValueError('bytes offset %g is greater than file size %g' %
                         (offset, actual_number_of_bytes))

    with open(datafile, 'rb') as f:
        if use_mmap:
            data = np.memmap(f, dtype=dtype, mode='r', offset=offset,
                             shape=tuple(shape), order=order)
        else:
            f.seek(offset)
            data = np.fromfile(f, dtype=dtype, count=number_of_values)
            data = data.reshape(shape, order=order)
    data.shape = shape
    return data 

def read_midc_raw_data_from_nrel(site, start, end, variable_map={},
                                 timeout=30):
    """Request and read MIDC data directly from the raw data api.

    Parameters
    ----------
    site: string
        The MIDC station id.
    start: datetime
        Start date for requested data.
    end: datetime
        End date for requested data.
    variable_map: dict
        A dictionary mapping MIDC field names to pvlib names. Used to
        rename columns of the resulting DataFrame. See Notes of
        :py:func:`pvlib.iotools.read_midc` for example.
    timeout : float, default 30
        Number of seconds to wait to connect/read from the API before
        failing.

    Returns
    -------
    data:
        Dataframe with DatetimeIndex localized to the station location.

    Raises
    ------
    requests.HTTPError
       For any error in retrieving the CSV file from the MIDC API
    requests.Timeout
       If data is not received in within ``timeout`` seconds

    Notes
    -----
    Requests spanning an instrumentation change will yield an error. See the
    MIDC raw data api page
    `here <https://midcdmz.nrel.gov/apps/data_api_doc.pl?_idtextlist>`_
    for more details and considerations.
    """
    args = {'site': site,
            'begin': start.strftime('%Y%m%d'),
            'end': end.strftime('%Y%m%d')}
    url = 'https://midcdmz.nrel.gov/apps/data_api.pl'
    # NOTE: just use requests.get(url, params=args) to build querystring
    # number of header columns and data columns do not always match,
    # so first parse the header to determine the number of data columns
    # to parse
    csv_request = requests.get(url, timeout=timeout, params=args)
    csv_request.raise_for_status()
    raw_csv = io.StringIO(csv_request.text)
    first_row = pd.read_csv(raw_csv, nrows=0)
    col_length = len(first_row.columns)
    raw_csv.seek(0)
    return read_midc(raw_csv, variable_map=variable_map, raw_data=True,
                     usecols=range(col_length)) 

def readDataRaw(self, request='', Debug_Mode=2):

        # self.testForEvents()                # First check for any pending events received

        tries = self.retries

        while tries > 0:
            try:
                if Debug_Mode >= 2:
                    logging.debug(str(len(request)) + "->   " + " ".join(hex(ord(i)) for i in request))
                self.sendData(request)
                inc_data = self.comms.recv(1024)
                if Debug_Mode >= 2:
                    logging.debug( str(len(inc_data)) + "<-   " + " ".join(hex(ord(i)) for i in inc_data))
                tries = 0

            except socket.timeout, e:
                err = e.args[0]
                if err == 'timed out':
                    #logging.error("Timed out error, no retry -<-- could fix this" + repr(e))
                    #this seems to be where it goes normally while waiting for traffic.
                    tries = 0
                    sys.exc_clear()
                    return ''
                    # sleep(1)
                    # print 'Receive timed out, ret'
                    # continue
                else:
                    logging.error("Error reading data from IP module, retrying again... (" + str(tries) + "): " + repr(e))
                    tries -= 1
                    time.sleep(Error_Delay)
                    sys.exc_clear()
                    pass
            except socket.error, e:
                logging.error("Unknown error on socket connection, retrying (%d) ... %s " % (tries, repr(e)))
                tries -= 1
                time.sleep(Error_Delay)
                if tries == 0:
                    logging.info("Failure, disconnected.")
                    sys.exit(1)
                else:
                    logging.error("After error, continuing %d attempts left" % tries)
                    sys.exc_clear()
                    return ''
                    continue