Skip to content

Commit

Permalink
More precise testing of plugins
Browse files Browse the repository at this point in the history
Instead of relying on a count of the severity and confidence
levels found within an example file, make use of Python's native
unit testing to verify the results of a plugin.

The existing method of confirming counts can be inaccurate. It's
very easy to have a false positive simply because one issue extra
was found and one issue was missed, thus giving the same count. It
tells nothing of the validation of a particular line of problematic
code.

Relates to #352

Signed-off-by: Eric Brown <[email protected]>
  • Loading branch information
ericwb committed Apr 7, 2023
1 parent 36fc7be commit 1c33807
Show file tree
Hide file tree
Showing 29 changed files with 6,486 additions and 535 deletions.
536 changes: 1 addition & 535 deletions tests/functional/test_functional.py

Large diffs are not rendered by default.

Empty file added tests/unit/plugins/__init__.py
Empty file.
33 changes: 33 additions & 0 deletions tests/unit/plugins/base_test_case.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# SPDX-License-Identifier: Apache-2.0
import testtools

from bandit.core import config
from bandit.core import manager
from bandit.core import meta_ast
from bandit.core import metrics
from bandit.core import node_visitor
from bandit.core import test_set


class BaseTestCase(testtools.TestCase):
def setUp(self, test_ids):
super().setUp()
b_config = config.BanditConfig()
self.b_manager = manager.BanditManager(b_config, "file")
issue_metrics = metrics.Metrics()
issue_metrics.begin("test.py")
self.visitor = node_visitor.BanditNodeVisitor(
"test.py",
None,
metaast=meta_ast.BanditMetaAst(),
testset=test_set.BanditTestSet(
b_config,
profile={
"include": test_ids,
"exclude": [],
},
),
debug=False,
nosec_lines={},
metrics=issue_metrics,
)
71 changes: 71 additions & 0 deletions tests/unit/plugins/test_app_debug.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# SPDX-License-Identifier: Apache-2.0
import textwrap

import bandit
from bandit.core import issue as b_issue
from tests.unit.plugins import base_test_case


class FlaskDebugTests(base_test_case.BaseTestCase):
def setUp(self):
super().setUp(["B201"])

def test_app_run_debug_true(self):
fdata = textwrap.dedent(
"""
from flask import Flask
app = Flask(__name__)
app.run(debug=True)
"""
)
self.visitor.process(fdata)
self.assertEqual(1, len(self.visitor.tester.results))
issue = self.visitor.tester.results[0]
self.assertEqual(bandit.HIGH, issue.severity)
self.assertEqual(bandit.MEDIUM, issue.confidence)
self.assertEqual(b_issue.Cwe.CODE_INJECTION, issue.cwe.id)
self.assertEqual(4, issue.lineno)
self.assertEqual([4], issue.linerange)
self.assertEqual(0, issue.col_offset)

def test_app_run_debug_false(self):
fdata = textwrap.dedent(
"""
from flask import Flask
app = Flask(__name__)
app.run(debug=False)
"""
)
self.visitor.process(fdata)
self.assertEqual(0, len(self.visitor.tester.results))

def test_app_run(self):
fdata = textwrap.dedent(
"""
from flask import Flask
app = Flask(__name__)
app.run()
"""
)
self.visitor.process(fdata)
self.assertEqual(0, len(self.visitor.tester.results))

def test_app_run_no_import(self):
fdata = textwrap.dedent(
"""
app = Flask(__name__)
app.run(debug=True)
"""
)
self.visitor.process(fdata)
self.assertEqual(0, len(self.visitor.tester.results))

def test_unrelated_run(self):
fdata = textwrap.dedent(
"""
from flask import Flask
run(debug=True)
"""
)
self.visitor.process(fdata)
self.assertEqual(0, len(self.visitor.tester.results))
23 changes: 23 additions & 0 deletions tests/unit/plugins/test_asserts.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# SPDX-License-Identifier: Apache-2.0
import bandit
from bandit.core import issue as b_issue
from tests.unit.plugins import base_test_case


class AssertsTests(base_test_case.BaseTestCase):
def setUp(self):
super().setUp(["B101"])

def test_asserts(self):
fdata = "assert True"
self.visitor.process(fdata)
self.assertEqual(1, len(self.visitor.tester.results))
issue = self.visitor.tester.results[0]
self.assertEqual(bandit.LOW, issue.severity)
self.assertEqual(bandit.HIGH, issue.confidence)
self.assertEqual(
b_issue.Cwe.IMPROPER_CHECK_OF_EXCEPT_COND, issue.cwe.id
)
self.assertEqual(1, issue.lineno)
self.assertEqual([1], issue.linerange)
self.assertEqual(0, issue.col_offset)
Loading

0 comments on commit 1c33807

Please sign in to comment.