Open
Description
List of test runs that expose a non-deterministic test result (failure) when run with a specific random seed (ordering).
To reproduce, use TESTARGS="--randomly-seed=<SEED> -vvv" make tests before adding to this list.
Seeds:
- 105155306, 1843708888, 19699634 -
7d32abd(#15469) - 3227833760, 235656747 -
15d3495(#15469) - 2329742959 -
1f0aff8(#15469) - 3916416581, 3294924425, 1890731092 - test: fix sort order #15747
- 46468882, 1997427500 - test: compare output by ids instead of object #16577
- 1697538841
- 2228510538
- 1862154588
- 816117609
Original issue:
To reproduce:
TESTARGS="--randomly-seed=105155306 -vvv" make tests T=tests/unit/api/test_simple.py::TestSimpleDetail(The T parameter is the narrowest scoping of tests I found that still exhibited the issue to save on a full test run.)
Other seeds that fail the same test:
- 1843708888
- 19699634
FAILED tests/unit/api/test_simple.py::TestSimpleDetail::test_with_files_no_serial[text/html-None] - AssertionError: assert equals failed
Failure:
========================================================================= FAILURES =========================================================================
________________________________________________ TestSimpleDetail.test_with_files_no_serial[text/html-None] ________________________________________________
self = <tests.unit.api.test_simple.TestSimpleDetail object at 0xffffa39bc310>, db_request = <pyramid.testing.DummyRequest object at 0xffff9ae9f310>
content_type = 'text/html', renderer_override = None
@pytest.mark.parametrize(
"content_type,renderer_override",
CONTENT_TYPE_PARAMS,
)
def test_with_files_no_serial(self, db_request, content_type, renderer_override):
db_request.accept = content_type
project = ProjectFactory.create()
releases = ReleaseFactory.create_batch(3, project=project)
release_versions = sorted([r.version for r in releases], key=parse)
files = [
FileFactory.create(release=r, filename=f"{project.name}-{r.version}.tar.gz")
for r in releases
]
# let's assert the result is ordered by string comparison of filename
files = sorted(files, key=lambda key: key.filename)
urls_iter = (f"/file/{f.filename}" for f in files)
db_request.matchdict["name"] = project.normalized_name
db_request.route_url = lambda *a, **kw: next(urls_iter)
user = UserFactory.create()
JournalEntryFactory.create(submitted_by=user)
> assert simple.simple_detail(project, db_request) == {
"meta": {"_last-serial": 0, "api-version": API_VERSION},
"name": project.normalized_name,
"versions": release_versions,
"files": [
{
"filename": f.filename,
"url": f"/file/{f.filename}",
"hashes": {"sha256": f.sha256_digest},
"requires-python": f.requires_python,
"yanked": False,
"size": f.size,
"upload-time": f.upload_time.isoformat() + "Z",
"data-dist-info-metadata": False,
"core-metadata": False,
}
for f in files
],
}
E AssertionError: assert equals failed
E { {
E 'files': [ 'files': [
E {
E 'core-metadata': False,
E 'data-dist-info-metadata': False,
E 'filename': 'LWlHXDYZQIKh-10.0.tar.gz',
E 'hashes': {
E 'sha256': 'd3a2990d1fc4c91a16eeb880dbd04f37bf645fb5df36126179
E b7dffb692a5fe2',
E },
E 'requires-python': None,
E 'size': 5823,
E 'upload-time': '2017-03-01T12:55:14.698742Z',
E 'url': '/file/LWlHXDYZQIKh-10.0.tar.gz',
E 'yanked': False,
E },
E { {
E 'core-metadata': False, 'core-metadata': False,
E 'data-dist-info-metadata': False, 'data-dist-info-metadata': False,
E 'filename': 'LWlHXDYZQIKh-8.0.tar.gz', 'filename': 'LWlHXDYZQIKh-8.0.tar.gz',
E 'hashes': { 'hashes': {
E 'sha256': '7b791e0d2c30b12063115d379d6c588758a4147b41b2a3c509 'sha256': '7b791e0d2c30b12063115d379d6c588758a4147b41b2a3c509
E a38b1c76d955cf', a38b1c76d955cf',
E }, },
E 'requires-python': None, 'requires-python': None,
E 'size': 7891, 'size': 7891,
E 'upload-time': '2009-02-12T05:59:26.341814Z', 'upload-time': '2009-02-12T05:59:26.341814Z',
E 'url': '/file/LWlHXDYZQIKh-10.0.tar.gz', 'url': '/file/LWlHXDYZQIKh-8.0.tar.gz',
E 'yanked': False, 'yanked': False,
E }, },
E { {
E 'core-metadata': False, 'core-metadata': False,
E 'data-dist-info-metadata': False, 'data-dist-info-metadata': False,
E --- ---
E 'sha256': '8e4adef422c7671d4e50db09cc893d9e3cc77610429dd68cf8 'sha256': '8e4adef422c7671d4e50db09cc893d9e3cc77610429dd68cf8
E 2728f00c2783b8', 2728f00c2783b8',
E }, },
E 'requires-python': None, 'requires-python': None,
E 'size': 2104, 'size': 2104,
E 'upload-time': '2019-03-22T08:14:52.990585Z', 'upload-time': '2019-03-22T08:14:52.990585Z',
E 'url': '/file/LWlHXDYZQIKh-8.0.tar.gz',
E 'yanked': False,
E },
E {
E 'core-metadata': False,
E 'data-dist-info-metadata': False,
E 'filename': 'LWlHXDYZQIKh-10.0.tar.gz',
E 'hashes': {
E 'sha256': 'd3a2990d1fc4c91a16eeb880dbd04f37bf645fb5df36126179
E b7dffb692a5fe2',
E },
E 'requires-python': None,
E 'size': 5823,
E 'upload-time': '2017-03-01T12:55:14.698742Z',
E 'url': '/file/LWlHXDYZQIKh-9.0.tar.gz', 'url': '/file/LWlHXDYZQIKh-9.0.tar.gz',
E 'yanked': False, 'yanked': False,
E }, },
E ], ],
E 'meta': {'_last-serial': 0, 'api-version': '1.1'}, 'meta': {'_last-serial': 0, 'api-version': '1.1'},
tests/unit/api/test_simple.py:261: AssertionError
Source test here:
warehouse/tests/unit/api/test_simple.py
Lines 240 to 286 in 38ba926
However it is likely something about the setup/teardown of other tests is mutating some state that's being left over.
P.S. There's a smattering of statements that don't actually assert, might be worth fixing once this test passes.
Example:
warehouse/tests/unit/api/test_simple.py
Lines 285 to 286 in 38ba926
Note the lack of a leading assert.