Python sqlalchemy.types.LargeBinary() Examples

def test_large_type_deprecation(self):
        d1 = mssql.dialect(deprecate_large_types=True)
        d2 = mssql.dialect(deprecate_large_types=False)
        d3 = mssql.dialect()
        d3.server_version_info = (11, 0)
        d3._setup_version_attributes()
        d4 = mssql.dialect()
        d4.server_version_info = (10, 0)
        d4._setup_version_attributes()

        for dialect in (d1, d3):
            eq_(str(Text().compile(dialect=dialect)), "VARCHAR(max)")
            eq_(str(UnicodeText().compile(dialect=dialect)), "NVARCHAR(max)")
            eq_(str(LargeBinary().compile(dialect=dialect)), "VARBINARY(max)")

        for dialect in (d2, d4):
            eq_(str(Text().compile(dialect=dialect)), "TEXT")
            eq_(str(UnicodeText().compile(dialect=dialect)), "NTEXT")
            eq_(str(LargeBinary().compile(dialect=dialect)), "IMAGE") 

def test_max_ident_in_varchar_not_present(self):
        """test [ticket:3504].

        Here we are testing not just that the "max" token comes back
        as None, but also that these types accept "max" as the value
        of "length" on construction, which isn't a directly documented
        pattern however is likely in common use.

        """
        metadata = self.metadata

        Table(
            "t",
            metadata,
            Column("t1", types.String),
            Column("t2", types.Text("max")),
            Column("t3", types.Text("max")),
            Column("t4", types.LargeBinary("max")),
            Column("t5", types.VARBINARY("max")),
        )
        metadata.create_all()
        for col in inspect(testing.db).get_columns("t"):
            is_(col["type"].length, None)
            in_("max", str(col["type"].compile(dialect=testing.db.dialect))) 

def test_comparison(self, connection):
        """test that type coercion occurs on comparison for binary"""

        expr = binary_table.c.data == "foo"
        assert isinstance(expr.right.type, LargeBinary)

        data = os.urandom(32)
        connection.execute(binary_table.insert(), data=data)
        eq_(
            connection.scalar(
                select([func.count("*")])
                .select_from(binary_table)
                .where(binary_table.c.data == data)
            ),
            1,
        ) 

def get_type(data_type):
    type_map = {
        "bytes": types.LargeBinary,
        "boolean": types.Boolean,
        "date": types.Date,
        "datetime": types.DateTime,
        "double": types.Numeric,
        "text": types.String,
        "keyword": types.String,
        "integer": types.Integer,
        "half_float": types.Float,
        "geo_point": types.String,
        # TODO get a solution for nested type
        "nested": types.String,
        # TODO get a solution for object
        "object": types.BLOB,
        "long": types.BigInteger,
        "float": types.Float,
        "ip": types.String,
    }
    type_ = type_map.get(data_type)
    if not type_:
        logger.warning(f"Unknown type found {data_type} reverting to string")
        type_ = types.String
    return type_ 

def test_binary(self):
        "Exercise type specification for binary types."

        columns = [
            # column type, args, kwargs, expected ddl
            (mssql.MSBinary, [], {}, "BINARY"),
            (mssql.MSBinary, [10], {}, "BINARY(10)"),
            (types.BINARY, [], {}, "BINARY"),
            (types.BINARY, [10], {}, "BINARY(10)"),
            (mssql.MSVarBinary, [], {}, "VARBINARY(max)"),
            (mssql.MSVarBinary, [10], {}, "VARBINARY(10)"),
            (types.VARBINARY, [10], {}, "VARBINARY(10)"),
            (types.VARBINARY, [], {}, "VARBINARY(max)"),
            (mssql.MSImage, [], {}, "IMAGE"),
            (mssql.IMAGE, [], {}, "IMAGE"),
            (types.LargeBinary, [], {}, "IMAGE"),
        ]

        metadata = MetaData()
        table_args = ["test_mssql_binary", metadata]
        for index, spec in enumerate(columns):
            type_, args, kw, res = spec
            table_args.append(
                Column("c%s" % index, type_(*args, **kw), nullable=None)
            )
        binary_table = Table(*table_args)
        dialect = mssql.dialect()
        gen = dialect.ddl_compiler(dialect, schema.CreateTable(binary_table))
        for col in binary_table.c:
            index = int(col.name[1:])
            testing.eq_(
                gen.get_column_specification(col),
                "%s %s" % (col.name, columns[index][3]),
            )
            self.assert_(repr(col)) 

def test_large_binary(self):
        stream1 = self._load_stream("binary_data_one.dat")
        self._test_round_trip(sqltypes.LargeBinary, stream1) 

def test_large_legacy_types(self):
        stream1 = self._load_stream("binary_data_one.dat")
        self._test_round_trip(
            sqltypes.LargeBinary, stream1, deprecate_large_types=False
        ) 

def test_python_type(self):
        eq_(types.Integer().python_type, int)
        eq_(types.Numeric().python_type, decimal.Decimal)
        eq_(types.Numeric(asdecimal=False).python_type, float)
        eq_(types.LargeBinary().python_type, util.binary_type)
        eq_(types.Float().python_type, float)
        eq_(types.Interval().python_type, datetime.timedelta)
        eq_(types.Date().python_type, datetime.date)
        eq_(types.DateTime().python_type, datetime.datetime)
        eq_(types.String().python_type, str)
        eq_(types.Unicode().python_type, util.text_type)
        eq_(types.Enum("one", "two", "three").python_type, str)

        assert_raises(
            NotImplementedError, lambda: types.TypeEngine().python_type
        ) 

def setup_class(cls):
        global binary_table, MyPickleType, metadata

        class MyPickleType(types.TypeDecorator):
            impl = PickleType

            def process_bind_param(self, value, dialect):
                if value:
                    value.stuff = "this is modified stuff"
                return value

            def process_result_value(self, value, dialect):
                if value:
                    value.stuff = "this is the right stuff"
                return value

        metadata = MetaData(testing.db)
        binary_table = Table(
            "binary_table",
            metadata,
            Column(
                "primary_id",
                Integer,
                primary_key=True,
                test_needs_autoincrement=True,
            ),
            Column("data", LargeBinary),
            Column("data_slice", LargeBinary(100)),
            Column("misc", String(30)),
            Column("pickled", PickleType),
            Column("mypickle", MyPickleType),
        )
        metadata.create_all() 

def test_bind_processor_no_dbapi(self):
        b = LargeBinary()
        eq_(b.bind_processor(default.DefaultDialect()), None) 

def open_database(context):
    """Opens the database specified in the feature file and creates
    tables if not already created

    :param context: The Behave context
    :return: A database handle, or None if no database in use
    """
    if hasattr(context, 'dburl') is False:
        return None  # No false positives database is in use
    dbconn = None

    # Try to connect to the database
    try:
        db_engine = create_engine(context.dburl)
        dbconn = db_engine.connect()
    except (IOError, exc.OperationalError):
        assert False, "Cannot connect to database '%s'" % context.dburl

    # Set up the database table to store new findings and false positives.
    # We use LargeBinary to store those fields that could contain somehow
    # bad Unicode, just in case some component downstream tries to parse
    # a string provided as Unicode.
    db_metadata = MetaData()
    db_metadata.bind = db_engine
    context.httpfuzzer_issues = Table('httpfuzzer_issues', db_metadata,
                                      Column('new_issue', types.Boolean),
                                      Column('issue_no', types.Integer, primary_key=True, nullable=False),
                                      Column('timestamp', types.DateTime(timezone=True)),
                                      Column('test_runner_host', types.Text),
                                      Column('scenario_id', types.Text),
                                      Column('url', types.Text),
                                      Column('server_protocol_error', types.Text),
                                      Column('server_timeout', types.Boolean),
                                      Column('server_error_text_detected', types.Boolean),
                                      Column('server_error_text_matched', types.Text),
                                      Column('req_method', types.Text),
                                      Column('req_headers', types.LargeBinary),
                                      Column('req_body', types.LargeBinary),
                                      Column('resp_statuscode', types.Text),
                                      Column('resp_headers', types.LargeBinary),
                                      Column('resp_body', types.LargeBinary),
                                      Column('resp_history', types.LargeBinary))

    # Create the table if it doesn't exist
    # and otherwise no effect
    db_metadata.create_all(db_engine)

    return dbconn 

def test_add_false_positive(self):
        # Add a false positive to database and check that all fields
        # get populated and can be compared back originals
        response = {'scenario_id': '1',
                    'req_headers': 'headers',
                    'req_body': 'body',
                    'url': 'url',
                    'timestamp': datetime.datetime.utcnow(),
                    'req_method': 'method',
                    'server_protocol_error': None,
                    'server_timeout': False,
                    'server_error_text_detected': False,
                    'server_error_text_matched': 'matched_text',
                    'resp_statuscode': 'statuscode',
                    'resp_headers': 'resp_headers',
                    'resp_body': 'resp_body',
                    'resp_history': 'resp_history'}

        dbtools.add_false_positive(self.context, response)

        # Connect directly to the database and check the data is there
        db_engine = sqlalchemy.create_engine(self.context.dburl)
        dbconn = db_engine.connect()
        db_metadata = sqlalchemy.MetaData()
        httpfuzzer_issues = Table('httpfuzzer_issues', db_metadata,
                                  Column('new_issue', types.Boolean),
                                  Column('issue_no', types.Integer, primary_key=True, nullable=False),
                                  Column('timestamp', types.DateTime(timezone=True)),
                                  Column('test_runner_host', types.Text),
                                  Column('scenario_id', types.Text),
                                  Column('url', types.Text),
                                  Column('server_protocol_error', types.Text),
                                  Column('server_timeout', types.Boolean),
                                  Column('server_error_text_detected', types.Boolean),
                                  Column('server_error_text_matched', types.Text),
                                  Column('req_method', types.Text),
                                  Column('req_headers', types.LargeBinary),
                                  Column('req_body', types.LargeBinary),
                                  Column('resp_statuscode', types.Text),
                                  Column('resp_headers', types.LargeBinary),
                                  Column('resp_body', types.LargeBinary),
                                  Column('resp_history', types.LargeBinary))
        db_select = sqlalchemy.sql.select([httpfuzzer_issues])
        db_result = dbconn.execute(db_select)
        result = db_result.fetchone()
        for key, value in response.iteritems():
            self.assertEqual(result[key], value,
                             '%s not found in database after add' % key)
        self.assertEqual(result['test_runner_host'], socket.gethostbyname(socket.getfqdn()),
                         'Test runner host name not correct in database')
        self.assertLessEqual(result['timestamp'], datetime.datetime.utcnow(),
                             'Timestamp not correctly stored in database')
        dbconn.close() 

def open_database(context):
    """Opens the database specified in the feature file and creates
    tables if not already created

    :param context: The Behave context
    :return: A database handle, or None if no database in use
    """
    if hasattr(context, 'dburl') is False:
        return None  # No false positives database is in use
    dbconn = None

    # Try to connect to the database
    try:
        db_engine = create_engine(context.dburl)
        dbconn = db_engine.connect()
    except (IOError, exc.OperationalError):
        assert False, "Cannot connect to database '%s'" % context.dburl

    # Set up the database table to store new findings and false positives.
    # We use LargeBinary to store the message, because it can potentially
    # be big.
    db_metadata = MetaData()
    db_metadata.bind = db_engine
    context.headlessscanner_issues = Table(
        'headlessscanner_issues',
        db_metadata,
        Column('new_issue', types.Boolean),
        Column('issue_no', types.Integer, primary_key=True, nullable=False),  # Implicit autoincrement
        Column('timestamp', types.DateTime(timezone=True)),
        Column('test_runner_host', types.Text),
        Column('scenario_id', types.Text),
        Column('url', types.Text),
        Column('severity', types.Text),
        Column('issuetype', types.Text),
        Column('issuename', types.Text),
        Column('issuedetail', types.Text),
        Column('confidence', types.Text),
        Column('host', types.Text),
        Column('port', types.Text),
        Column('protocol', types.Text),
        Column('messages', types.LargeBinary)
    )

    # Create the table if it doesn't exist
    # and otherwise no effect
    db_metadata.create_all(db_engine)

    return dbconn 

def test_add_false_positive(self):
        # Add a false positive to database and check that all fields
        # get populated and can be compared back originals
        issue = {'scenario_id': '1',
                 'url': 'testurl',
                 'severity': 'testseverity',
                 'issuetype': 'testissuetype',
                 'issuename': 'testissuename',
                 'issuedetail': 'testissuedetail',
                 'confidence': 'testconfidence',
                 'host': 'testhost',
                 'port': 'testport',
                 'protocol': 'testprotocol',
                 'messages': '{foo=bar}'}

        dbtools.add_false_positive(self.context, issue)

        # Connect directly to the database and check the data is there
        db_engine = sqlalchemy.create_engine(self.context.dburl)
        dbconn = db_engine.connect()
        db_metadata = sqlalchemy.MetaData()
        headlessscanner_issues = Table(
            'headlessscanner_issues',
            db_metadata,
            Column('new_issue', types.Boolean),
            Column('issue_no', types.Integer, primary_key=True, nullable=False),  # Implicit autoincrement
            Column('timestamp', types.DateTime(timezone=True)),
            Column('test_runner_host', types.Text),
            Column('scenario_id', types.Text),
            Column('url', types.Text),
            Column('severity', types.Text),
            Column('issuetype', types.Text),
            Column('issuename', types.Text),
            Column('issuedetail', types.Text),
            Column('confidence', types.Text),
            Column('host', types.Text),
            Column('port', types.Text),
            Column('protocol', types.Text),
            Column('messages', types.LargeBinary)
        )
        db_select = sqlalchemy.sql.select([headlessscanner_issues])
        db_result = dbconn.execute(db_select)
        result = db_result.fetchone()
        for key, value in issue.iteritems():
            if key == 'messages':
                self.assertEqual(result[key], json.dumps(value))
            else:
                self.assertEqual(result[key], value,
                                 '%s not found in database after add' % key)
        self.assertEqual(result['test_runner_host'], socket.gethostbyname(socket.getfqdn()),
                         'Test runner host name not correct in database')
        self.assertLessEqual(result['timestamp'], datetime.datetime.utcnow(),
                             'Timestamp not correctly stored in database')
        dbconn.close() 

def _test_binary_reflection(self, deprecate_large_types):
        "Exercise type specification for binary types."

        columns = [
            # column type, args, kwargs, expected ddl from reflected
            (mssql.MSBinary, [], {}, "BINARY(1)"),
            (mssql.MSBinary, [10], {}, "BINARY(10)"),
            (types.BINARY, [], {}, "BINARY(1)"),
            (types.BINARY, [10], {}, "BINARY(10)"),
            (mssql.MSVarBinary, [], {}, "VARBINARY(max)"),
            (mssql.MSVarBinary, [10], {}, "VARBINARY(10)"),
            (types.VARBINARY, [10], {}, "VARBINARY(10)"),
            (types.VARBINARY, [], {}, "VARBINARY(max)"),
            (mssql.MSImage, [], {}, "IMAGE"),
            (mssql.IMAGE, [], {}, "IMAGE"),
            (
                types.LargeBinary,
                [],
                {},
                "IMAGE" if not deprecate_large_types else "VARBINARY(max)",
            ),
        ]

        metadata = self.metadata
        metadata.bind = engines.testing_engine(
            options={"deprecate_large_types": deprecate_large_types}
        )
        table_args = ["test_mssql_binary", metadata]
        for index, spec in enumerate(columns):
            type_, args, kw, res = spec
            table_args.append(
                Column("c%s" % index, type_(*args, **kw), nullable=None)
            )
        binary_table = Table(*table_args)
        metadata.create_all()
        reflected_binary = Table(
            "test_mssql_binary", MetaData(testing.db), autoload=True
        )
        for col, spec in zip(reflected_binary.c, columns):
            eq_(
                str(col.type),
                spec[3],
                "column %s %s != %s" % (col.key, str(col.type), spec[3]),
            )
            c1 = testing.db.dialect.type_descriptor(col.type).__class__
            c2 = testing.db.dialect.type_descriptor(
                binary_table.c[col.name].type
            ).__class__
            assert issubclass(
                c1, c2
            ), "column %s: %r is not a subclass of %r" % (col.key, c1, c2)
            if binary_table.c[col.name].type.length:
                testing.eq_(
                    col.type.length, binary_table.c[col.name].type.length
                ) 

def test_round_trip(self, connection):
        testobj1 = pickleable.Foo("im foo 1")
        testobj2 = pickleable.Foo("im foo 2")
        testobj3 = pickleable.Foo("im foo 3")

        stream1 = self.load_stream("binary_data_one.dat")
        stream2 = self.load_stream("binary_data_two.dat")
        connection.execute(
            binary_table.insert(),
            primary_id=1,
            misc="binary_data_one.dat",
            data=stream1,
            data_slice=stream1[0:100],
            pickled=testobj1,
            mypickle=testobj3,
        )
        connection.execute(
            binary_table.insert(),
            primary_id=2,
            misc="binary_data_two.dat",
            data=stream2,
            data_slice=stream2[0:99],
            pickled=testobj2,
        )
        connection.execute(
            binary_table.insert(),
            primary_id=3,
            misc="binary_data_two.dat",
            data=None,
            data_slice=stream2[0:99],
            pickled=None,
        )

        for stmt in (
            binary_table.select(order_by=binary_table.c.primary_id),
            text(
                "select * from binary_table order by binary_table.primary_id",
                bind=testing.db,
            ).columns(
                **{
                    "pickled": PickleType,
                    "mypickle": MyPickleType,
                    "data": LargeBinary,
                    "data_slice": LargeBinary,
                }
            ),
        ):
            result = connection.execute(stmt).fetchall()
            eq_(stream1, result[0]._mapping["data"])
            eq_(stream1[0:100], result[0]._mapping["data_slice"])
            eq_(stream2, result[1]._mapping["data"])
            eq_(testobj1, result[0]._mapping["pickled"])
            eq_(testobj2, result[1]._mapping["pickled"])
            eq_(testobj3.moredata, result[0]._mapping["mypickle"].moredata)
            eq_(
                result[0]._mapping["mypickle"].stuff, "this is the right stuff"
            )