Python get stock data

def get_stock_info_and_historical_data(symbol):
    """Retrieve stock info and historical data for `symbol`

    Leverages yahoo-finance for Python
    https://github.com/lukaszbanasiak/yahoo-finance
    """
    share = yahoo_finance.Share(symbol)
    info = share.get_info()
    start_date = info['start']
    end_date = info['end']
    historical = share.get_historical(start_date, end_date)
    data = {
        'info' : info,
        'historical' : historical,
    }
    return data 

def getStockData(theTicker, startDate):
    stock = Share(theTicker)
    print("Getting Data for ... " + theTicker)
    now = datetime.now()
    DateNow = str(now.year) + "-" + str(now.month) + "-" + str(now.day)
    data = stock.get_historical(startDate, DateNow)
    stockData = []
    for d in data:
        tmp = []
        volume = int(d['Volume'])
        adjclose = float(d['Adj_Close'])
        high = float(d['High'])
        low = float(d['Low'])
        close = float(d['Close'])
        date = d['Date']
        open = float(d['Open'])
        tmp.append(date)
        tmp.append(open)
        tmp.append(high)
        tmp.append(low)
        tmp.append(close)
        tmp.append(adjclose)
        tmp.append(volume)
        stockData.append(tmp)
    return stockData 

def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}):
        data = {}
        if isinstance(self.ticker, str):
            dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj)
            data.update(dict_ent)
        else:
            for tick in self.ticker:
                try:
                    dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj)
                    data.update(dict_ent)
                except ManagedException:
                    print("Warning! Ticker: " + str(tick) + " error - " + str(ManagedException))
                    print("The process is still running...")
                    continue
        return data

    # Public Method to get technical stock data 

def get_stock_dividend_data(self, start, end, interval):
        interval_code = self.get_time_code(interval)
        if isinstance(self.ticker, str):
            try:
                return {self.ticker: self._handle_api_dividend_request(self.ticker, start, end, interval_code)}
            except:
                return {self.ticker: None}
        else:
            re_data = {}
            for tick in self.ticker:
                try:
                    div_data = self._handle_api_dividend_request(tick, start, end, interval_code)
                    re_data.update({tick: div_data})
                except:
                    re_data.update({tick: None})
            return re_data


# Class containing methods to create stock data extracts 

def get_stock_current_data(self, quote):
        stock_current_data = None
        base_url = "https://query.yahooapis.com/v1/public/yql"
        select_statement = 'select * from yahoo.finance.quote where symbol in ("{quote}")'.format(quote=quote)
        query = urllib.parse.quote(select_statement)
        result_format = 'json'
        env = urllib.parse.quote("store://datatables.org/alltableswithkeys")
        url = base_url + "?" + "q=" + query + "&format=" + result_format + "&env=" + env
        response = requests.get(url)
        status_code = response.status_code
        body = response.json()
        if status_code == 200:
            count = body["query"]["count"]
            if count > 0:
                stock_current_data = body["query"]["results"]["quote"]
                stock_current_data = json.json_keys_to_lower_and_snake_case(stock_current_data)
        else:
            logger.error("get_stock_current_data: status_code: %i, body: %s", status_code, body)
        return stock_current_data 

def get_stock_historical_data(self, initialDate, finalDate, quote):
        stock_historical_data_array = []
        base_url = "https://query.yahooapis.com/v1/public/yql"
        select_statement = 'select * from yahoo.finance.historicaldata where symbol = "{quote}" and startDate = "{initialDate}" and endDate = "{finalDate}"'.format(quote=quote, initialDate=initialDate, finalDate=finalDate)
        query = urllib.parse.quote(select_statement)
        result_format = 'json'
        env = urllib.parse.quote("store://datatables.org/alltableswithkeys")
        url = base_url + "?" + "q=" + query + "&format=" + result_format + "&env=" + env
        response = requests.get(url)
        status_code = response.status_code
        body = response.json()
        if status_code == 200:
            count = body["query"]["count"]
            if count > 0:
                stock_historical_data_array = body["query"]["results"]["quote"]
                stock_historical_data_array = json.json_keys_to_lower_and_snake_case(stock_historical_data_array)
        else:
            logger.error("get_stock_historical_data: status_code: %i, body: %s", status_code, body)
        return stock_historical_data_array 

def get_single_stock_data_monthly(root_path, symbol):
    '''
    All data is from quandl wiki dataset
    Feature set: [Open  High    Low  Close    Volume  Ex-Dividend  Split Ratio Adj. Open  Adj. High  Adj. Low
    Adj. Close  Adj. Volume]
    '''
    df, lastUpdateTime = queryStock(root_path, "DB_STOCK", "SHEET_CHN", "_MONTHLY", symbol, "monthly_update")
    df.index = pd.to_datetime(df.index)

    #suspended_day = pd.Timestamp((datetime.datetime.now() - datetime.timedelta(days=3)).strftime("%Y-%m-%d"))

    if df.empty: 
        #print("stock delisted", symbol)
        return df

    # if df.index[-1] < suspended_day:
    #     #print("stock suspended", symbol)
    #     return pd.DataFrame()

    return df 

def get_single_stock_data_daily(root_path, symbol):
    '''
    All data is from quandl wiki dataset
    Feature set: [Open  High    Low  Close    Volume  Ex-Dividend  Split Ratio Adj. Open  Adj. High  Adj. Low
    Adj. Close  Adj. Volume]
    '''
    df, lastUpdateTime = queryStock(root_path, "DB_STOCK", "SHEET_CHN", "_DAILY", symbol, "daily_update")
    df.index = pd.to_datetime(df.index)

    suspended_day = pd.Timestamp((datetime.datetime.now() - datetime.timedelta(days=3)).strftime("%Y-%m-%d"))

    if df.empty: 
        #print("stock delisted", symbol)
        return df

    if df.index[-1] < suspended_day:
        #print("stock suspended", symbol)
        return pd.DataFrame()

    return df 

def get_single_stock_data_weekly(root_path, symbol):
    '''
    All data is from quandl wiki dataset
    Feature set: [Open  High    Low  Close    Volume  Ex-Dividend  Split Ratio Adj. Open  Adj. High  Adj. Low
    Adj. Close  Adj. Volume]
    '''
    df, lastUpdateTime = queryStock(root_path, "DB_STOCK", "SHEET_CHN", "_WEEKLY", symbol, "weekly_update")
    df.index = pd.to_datetime(df.index)

    #suspended_day = pd.Timestamp((datetime.datetime.now() - datetime.timedelta(days=3)).strftime("%Y-%m-%d"))

    if df.empty: 
        #print("stock delisted", symbol)
        return df

    # if df.index[-1] < suspended_day:
    #     #print("stock suspended", symbol)
    #     return pd.DataFrame()

    return df 

def get_stock_data(self, jarvis, quote):
        ''' Given a stock symbol, get the real time price of the stock '''
        url = 'https://financialmodelingprep.com/api/v3/stock/real-time-price/' + quote
        resp = requests.get(url)

        if(resp.status_code == 200):
            data = resp.json()
            if('symbol' in data.keys()):
                jarvis.say("Symbol: " + str(data['symbol']), Fore.GREEN)
                jarvis.say("Price: " + str(data['price']), Fore.GREEN)
                jarvis.say("IEX Real-Time Price (https://iextrading.com/developer)")
            elif('Error' in data.keys()):
                jarvis.say("Invalid stock symbol name", Fore.RED)
            else:
                jarvis.say("Error. Please retry")
        else:
            jarvis.say("Cannot find the name. Try again later\n", Fore.RED) 

def GetStockData(self, stockList, symbolList):
        symbolString = self.__buildSymbolString(stockList)
        apiUrl = self.__buildUrl(symbolString)

        try:
            request = urllib2.urlopen(apiUrl)
            response = request.read()[3:-1].strip()
            request.close()
        except Exception as e:
            print(e)
            return ''

        jsonDataList = json.loads(response)
        stockInfoStr = self.__buildStockInfoStr(jsonDataList, symbolList)

        return stockInfoStr 

def get_stock_data(share, map_of_fields):
    share_map = share.__dict__['data_set']
    return DictParser.parse_dtypes({value: share_map.get(key, None)
                                    for key, value in
                                    map_of_fields.items()},
                                   yahoo_dtype_map,
                                   default=str,
                                   raise_on_error=False) 

def get_stock_data_historical(share, start_date, end_date):
    share_list = share.get_historical(start_date, end_date)
    return [DictParser.parse_dtypes({value: s.get(key, None)
                                     for key, value in
                                     yahoo_historical_map.items()},
                                    yahoo_dtype_map,
                                    default=str,
                                    raise_on_error=False)
            for s in share_list] 

def getStockHistDaysDataSourceFileName():
        path = DyCommon.createPath('Stock/User/Config/Common')
        file = os.path.join(path, 'DyStockHistDaysDataSource.json')

        return file 

def get_stock_data(ticker, start_date, end_date):
    """
    Gets historical stock data of given tickers between dates
    :param ticker: company, or companies whose data is to fetched
    :type ticker: string or list of strings
    :param start_date: starting date for stock prices
    :type start_date: string of date "YYYY-mm-dd"
    :param end_date: end date for stock prices
    :type end_date: string of date "YYYY-mm-dd"
    :return: stock_data.csv
    """
    i = 1
    try:
        all_data = pdr.get_data_yahoo(ticker, start_date, end_date)
    except ValueError:
        print("ValueError, trying again")
        i += 1
        if i < 5:
            time.sleep(10)
            get_stock_data(ticker, start_date, end_date)
        else:
            print("Tried 5 times, Yahoo error. Trying after 2 minutes")
            time.sleep(120)
            get_stock_data(ticker, start_date, end_date)
    stock_data = all_data["Adj Close"]
    stock_data.to_csv("stock_prices.csv") 

def getStockData(key):
    datavec = []
    lines = open("data/" + key + ".csv", "r").read().splitlines()
    
    for line in lines[1:]:
        datavec.append(float(line.split(",")[4]))
    
    return datavec 

def get_stock_basics_data(date=None):
    """
        获取沪深上市公司基本情况
    Parameters
    date:日期YYYY-MM-DD，默认为上一个交易日，目前只能提供2016-08-09之后的历史数据
    Return
    --------
    DataFrame
               code,代码
               name,名称
               industry,细分行业
               area,地区
               pe,市盈率
               outstanding,流通股本
               totals,总股本(万)
               totalAssets,总资产(万)
               liquidAssets,流动资产
               fixedAssets,固定资产
               reserved,公积金
               reservedPerShare,每股公积金
               eps,每股收益
               bvps,每股净资
               pb,市净率
               timeToMarket,上市日期
    """
    wdate = du.last_tddate() if date is None else date
    wdate = wdate.replace('-', '')
    if wdate < '20160809':
        return None
    datepre = '' if date is None else wdate[0:4] + wdate[4:6] + '/'
    request = Request(ct.ALL_STOCK_BASICS_FILE%(datepre, '' if date is None else wdate))
    text = urlopen(request, timeout=10).read()
    text = text.decode('GBK')
    text = text.replace('--', '')
    df = pd.read_csv(StringIO(text), dtype={'code':'object'})
    df = df.set_index('code')
    return df 

def get_stock_earnings_data(self, reformat=True):
        if reformat:
            return self.get_clean_data(self.get_stock_tech_data('earnings'), 'earnings')
        else:
            return self.get_stock_tech_data('earnings')

    # Public Method for the user to get stock summary data 

def get_stock_tech_data(self, tech_type):
        if tech_type == 'defaultKeyStatistics':
            return self.get_stock_data(statement_type='keystats', tech_type=tech_type)
        else:
            return self.get_stock_data(tech_type=tech_type)

    # Public Method to get reformatted statement data 

def get_stock_price_data(self, reformat=True):
        if reformat:
            return self.get_clean_data(self.get_stock_tech_data('price'), 'price')
        else:
            return self.get_stock_tech_data('price')

    # Public Method for the user to return key-statistics data 

def get_stock_data(url_address):
    """url_address로 Google 서버에 요청하면 해당 정보를 다운로드 받은후 Two Dimensional List 변환하여 반환함

    아래 코드를 활용하면 Google 서버에서 데이터를 String Type으로 다운로드 받을 수 있음

        >>> import urllib.request
        >>> url_address = 'http://finance.google.com/finance/historical?q=KRX:005930&startdate=2017-01-01&enddate=2017-12-30&output=csv'
        >>> r = urllib.request.urlopen(url_address)
        >>> stock_data_string = r.read().decode("utf8")    # String Type으로 다운로드 받은 데이터
        >>> stock_data_string[:50]
    'Date,Open,High,Low,Close,Volume\n28-Apr-17,2214.36'

    다운된 데이터는 String Type으로 각 줄은 "\n"으로 구분되며, 필드 데이터들은 ","으로 구분됨

    Args:
        url_address (str): Google 금융 데이터 서버에 주식 csv 데이터를 요청하는 URL 주소

    Returns:
        list: Two Dimensional List 0번째 index에는 필드의 Header 정보를 포함하고 있으며,
        1번째 index부터 각 필드의 data를 가지고 있음

    Examples:
        >>> import stock_data_crawler as sdc
        >>> url = 'http://finance.google.com/finance/historical?q=KRX:005930&startdate=2013-01-01&enddate=2015-12-30&output=csv'
        >>> sdc.get_stock_data(url)[:3]
        [['\ufeffDate', 'Open', 'High', 'Low', 'Close', 'Volume'], ['30-Dec-15', '1260000.00', '1272000.00', '1254000.00', '1260000.00', '203349'],
        ['29-Dec-15', '1265000.00', '1266000.00', '1241000.00', '1254000.00', '231802']]
    """
    r = urllib.request.urlopen(url_address)
    stock_data_string = r.read().decode("utf8").strip() # 반드시 Strip을 추가할 것
    # ===Modify codes below=============


    # ==================================
    return result 

def get_stock_data(self, force=False):
        # Update only once every 60 seconds
        if self.millis() - self.last_update < 6 * 1000 * 60 and not force:
            return False

        update = threading.Thread(None, self.do_update)
        update.daemon = True
        update.start() 

def get_single_stock_data(root_path, symbol, dates_range):
    
    df, lastUpdateTime = queryStock(root_path, "DB_STOCK", "SHEET_US", "_DAILY", symbol, "daily_update")
    if df.empty: return pd.DataFrame()
    
    df.index = pd.to_datetime(df.index)
    df = df[df.index.isin(dates_range)].sort_index()
    df.loc[:, 'Close_Shift_1'] = df.loc[:, 'adj_close'].shift(1)
    
    df.loc[:, 'Return'] = df.loc[:, 'adj_close']/df.loc[:, 'Close_Shift_1'] - 1
    return df 

def get_stock_tick_data(code=None, date=None, retry_count=100000, pause=0.1):
    """
    获取分笔数据
    Parameters
    ------
        code:string
                  股票代码 e.g. 600048
        date:string
                  日期 format：YYYY-MM-DD
        retry_count : int, 默认 10000
                  如遇网络等问题重复执行的次数
        pause : int, 默认 0.1
                  重复请求数据过程中暂停的秒数，防止请求间隔时间太短出现的问题
     return
     -------
        DataFrame 当日所有股票交易数据(DataFrame)
              属性:成交时间、成交价格、价格变动，成交手、成交金额(元)，买卖类型
    """
    if code is None or len(code) != 6 or date is None:
        return None
    symbol = _code_to_symbol(code)
    for _ in range(retry_count):
        time.sleep(pause)
        try:
            # http://market.finance.sina.com.cn/downxls.php?date=2016-10-28&symbol=600048
            #print str(ct.TICK_PRICE_URL % (ct.P_TYPE['http'], ct.DOMAINS['sf'], ct.PAGES['dl'],
            #                    date, symbol))
            re = Request(ct.TICK_PRICE_URL % (ct.P_TYPE['http'], ct.DOMAINS['sf'], ct.PAGES['dl'],
                                date, symbol))
            
            lines = urlopen(re, timeout=10).read()
            lines = lines.decode('GBK')
            if len(lines) < 20:
                return None
            df = pd.read_table(StringIO(lines), names=ct.TICK_COLUMNS, skiprows=[0])      
        except Exception as e:
            print(e)
        else:
            return df
    raise IOError(ct.NETWORK_URL_ERROR_MSG)


#----------------------------min----------------------------------------------------- 

def get_stock_min_data(code=None, date=None, ktype=1, retry_count=100000, pause=0.1):
    """
    获取分钟数据
    Parameters
    ------
        code:string
                  股票代码 e.g. 600048
        date:string
                  日期 format：YYYY-MM-DD
        retry_count : int, 默认 10000
                  如遇网络等问题重复执行的次数
        pause : int, 默认 0.1
                  重复请求数据过程中暂停的秒数，防止请求间隔时间太短出现的问题
     return
     -------
        DataFrame 当日所有股票交易数据(DataFrame)
              属性:成交日期、成交时间、open、high、low、close、成交手、成交金额(元)
    """
    if code is None or len(code) != 6 or date is None:
        return None
    # 获取股票的tick数据。
    # 数据属性分别为：时间、价格、价格变动，成交量、成交额和成交类型。
    df = get_stock_tick_data(code, date)
    if df.shape[0] == 3: # 当天没有数据
        return None

    # 数据清洗和准备
    # 将时间列加上日期，然后转换成datetime类型，并将其设置为index。
    df['time'] = date + ' ' + df['time']
    df['time'] = pd.to_datetime(df['time'])
    df = df.set_index('time')

    # 从一行行TICK的价格数据转化成开盘、最高、最低和收盘4个数据项。
    # 需要用到pandas的resample方法，并通过ohlc函数实现自动合成。
    min_type = str(ktype) + 'min'
    price_df = df['price'].resample(min_type).ohlc()
    price_df = price_df.dropna()

    # 成交量和成交额计算
    # 成交量和成交额是本分钟内的所有量的总和，所以会用到resample的sum函数。
    vols = df['volume'].resample(min_type).sum()
    vols = vols.dropna()
    vol_df = pd.DataFrame(vols, columns=['volume'])

    amounts = df['amount'].resample(min_type).sum()
    amounts = amounts.dropna()
    amount_df = pd.DataFrame(amounts, columns=['amount'])

    # 数据合并
    # 将价格数据与成交量成交额合并成一个完整的DataFrame。
    newdf = price_df.merge(vol_df, left_index=True, right_index=True).merge(amount_df, left_index=True, right_index=True)
    return newdf

    '''
    # 1分钟转N分钟
    d_dict = {
        'open':'first',
        'high':'max',
        'close':'last',
        'low':'min',
        'volume':'sum',
        'amount':'sum'
    }

    new = pd.DataFrame()
    for col in newdf.columns:
        new[col] = newdf[col].resample('5min', how=d_dict[col])
    '''




# ----------------------------help functions----------------------------------------- 

def get_single_stock_data_daily_date(root_path, symbol, start_date):
    '''
    All data is from quandl wiki dataset
    Feature set: [Open  High    Low  Close    Volume  Ex-Dividend  Split Ratio Adj. Open  Adj. High  Adj. Low
    Adj. Close  Adj. Volume]
    '''
    df = get_single_stock_data_daily(root_path, symbol)

    if df.empty: 
        #print("stock delisted", symbol)
        return pd.DataFrame(), pd.DataFrame(), []

    out_path = root_path + "/Data/CSV/cashflow/"
    exception_path = root_path + "/Data/CSV/exception/"
    
    if os.path.exists(out_path) == False:
        os.mkdir(out_path)
    if os.path.exists(exception_path) == False:
        os.mkdir(exception_path)

    out_file = out_path + symbol + ".csv"
    exception_file =exception_path + symbol + ".csv"

    df.index = pd.to_datetime(df.index)
    df = df[df.index >= pd.Timestamp(start_date)]
    datelist = df.index.strftime('%Y-%m-%d')

    if os.path.exists(exception_file):
        exception_df = pd.read_csv(exception_file, index_col=0)
        temp_df = exception_df[(exception_df['retry'] > 0)] #or (exception_df['last_update']=='True')]
        df_symbol_date_list = temp_df['date'].values.tolist()
        datelist = list( set(datelist) - set(df_symbol_date_list) )
    else:
        exception_df = pd.DataFrame(columns=['date', 'retry', 'last_update'])

    if os.path.exists(out_file) == False:
        df = pd.DataFrame(columns=['date', 'symbol', 'buy_amount', 'sell_amount', 'even_amount', 'buy_volume', 'sell_volume', 'even_volume', 'buy_max', 'buy_min', 'buy_average', 'sell_max', 'sell_min', 'sell_average', 'even_max', 'even_min', 'even_average'])
        df.index.name = 'index'
        return df, exception_df, datelist

    df = pd.read_csv(out_file, index_col=["index"])
    df_symbol_date_list = df['date'].values.tolist()
    #date_list = list( (set(date_list) | set(df_symbol_date_list)) - (set(date_list) & set(df_symbol_date_list)) )
    datelist = list( set(datelist) - set(df_symbol_date_list) )

    return df, exception_df, datelist 

def getStockDataVec(self, key):
        ''' 
        Open path and get data
            args:
                key: (str) data path
            return:
                vec: (pandas dataframe) price array
                row: (pandas dataframe) all array except time
                time: (pandas dataframe) time array
        '''
        path = key#"data/" + key + ".csv"
        if not os.path.exists(path):
            raise ValueError("Your stock {} is not in data/ directory.".format(key))
        vec = None
        row = None
        chunksize = 10000
        nlines = subprocess.check_output('wc -l %s' % path, shell=True)
        nlines = int(nlines.split()[0])
        chunksize = nlines // 10
        lines = subprocess.check_output('head %s' % path, shell=True).decode()
        lines = lines.split('\n')[0]

        if ',' in lines:
            sep = ','
            len_row = len(lines.split(sep))
            if len_row == 4:
                #types = dict(BID='np.float64', ASK='np.float64', Volume='np.float64')
                #names = ['ID', 'Time', 'Price', 'Volume']
                names = ['Time', 'BID', 'ASK', 'Volume']
            elif len_row == 3:
                #types = dict(Price='np.float64', Volume='np.float64')
                names = ['Time', 'Price', 'Volume']
            elif len_row == 6:
                names = ['Time', 'Price', 'Volume', 'RSI', 'MACD', 'Volatility']#, 'EMA20', 'EMA50', 'EMA100']
        elif ';' in lines:
            sep = ';'
            len_row = len(lines.split(sep))
            if len_row == 6:
                #types = dict(Open=np.float64, High=np.float64, Low=np.float64, Close=np.float64)
                names = ['Time', 'Open', 'High', 'Low', 'Close', '']

        for i in range(0, nlines, chunksize):
            df = pd.read_csv(path, header=None, sep=sep, nrows=chunksize,
                skiprows=i, low_memory=False)
            df.columns = names
            if row is not None:
                row = row.append(df, ignore_index=True)
            else:
                row = df.copy(deep=True)

        time = row['Time'].copy(deep=True)

        if len_row == 4 and ',' in sep:
            vec = row['ASK'].copy(deep=True)
        elif len_row == 3 and ',' in sep:
            vec = row['Price'].copy(deep=True)
        elif len_row == 6 and ',' in sep:
            vec = row['Price'].copy(deep=True)
        elif len_row == 6 and ';' in sep:
            vec = row['Close'].copy(deep=True)

        row.drop(row.columns[[0]], axis=1, inplace=True)
        return vec, row, time