Kiln » Dependencies » Dulwich Read More
Clone URL:  
Pushed to one repository · View In Graph Contained in master

Add cgit compatibility testing framework.

This adds a suite of tests that require git-core to be installed and can be run
with "make check-compat". These tests can run cgit via subprocess and capture
the results. This is primarily used to test the git and HTTP protocol servers
against their cgit counterparts, but other tests are possible as well. Also
included a test that packs written by dulwich are verified by git verify-pack.

The servers are tested by running the server in a separate thread and spawning a
git process that talks to them, then ensuring that the correct operations were
applied to each repo.

Also fixed/added in the course of testing:
-Fixed a bad merge in server.py
-Fixed some global namespace bugs in web.py
-Refresh the object store pack cache if the pack directory is modified.
-Added a 'dumb' flag to HTTPGitApplication so the HTTP server can be run in
dumb-only mode. This allows testing the dumb server against a smart cgit client
(which has no option to turn off smart HTTP).

There are still several outstanding bugs that cause tests to fail. The relevant
tests are currently skipped and marked with TODO.

Change-Id: I2b4fd0af6e59d03815ca663268441e5696883763

Changeset d1de8cd78da8

Parent e2d0a615baaa

committed by Dave Borowitz

authored by Dave Borowitz

Changes to 13 files · Browse files at d1de8cd78da8 Showing diff from parent e2d0a615baaa Diff from another changeset...

Change 1 of 2 Show Entire File Makefile Stacked
 
3
4
5
6
 
7
8
9
 
23
24
25
 
 
 
26
27
28
 
3
4
5
 
6
7
8
9
 
23
24
25
26
27
28
29
30
31
@@ -3,7 +3,7 @@
 PYDOCTOR ?= pydoctor  TESTRUNNER = $(shell which nosetests)   -all: build +all: build    doc:: pydoctor   @@ -23,6 +23,9 @@
 check-noextensions:: clean   PYTHONPATH=. $(PYTHON) $(TESTRUNNER) dulwich   +check-compat:: build + PYTHONPATH=. $(PYTHON) $(TESTRUNNER) -i compat +  clean::   $(SETUP) clean --all   rm -f dulwich/*.so
 
253
254
255
 
 
 
 
256
257
258
 
263
264
265
266
 
267
268
269
 
332
333
334
 
335
336
337
338
339
 
 
 
340
341
342
 
348
349
350
 
 
 
 
 
 
 
 
351
352
353
 
253
254
255
256
257
258
259
260
261
262
 
267
268
269
 
270
271
272
273
 
336
337
338
339
340
341
342
343
 
344
345
346
347
348
349
 
355
356
357
358
359
360
361
362
363
364
365
366
367
368
@@ -253,6 +253,10 @@
  def _load_packs(self):   raise NotImplementedError(self._load_packs)   + def _pack_cache_stale(self): + """Check whether the pack cache is stale.""" + raise NotImplementedError(self._pack_cache_stale) +   def _add_known_pack(self, pack):   """Add a newly appeared pack to the cache by path.   @@ -263,7 +267,7 @@
  @property   def packs(self):   """List with pack objects.""" - if self._pack_cache is None: + if self._pack_cache is None or self._pack_cache_stale():   self._pack_cache = self._load_packs()   return self._pack_cache   @@ -332,11 +336,14 @@
  super(DiskObjectStore, self).__init__()   self.path = path   self.pack_dir = os.path.join(self.path, PACKDIR) + self._pack_cache_time = 0     def _load_packs(self):   pack_files = []   try: - for name in os.listdir(self.pack_dir): + self._pack_cache_time = os.stat(self.pack_dir).st_mtime + pack_dir_contents = os.listdir(self.pack_dir) + for name in pack_dir_contents:   # TODO: verify that idx exists first   if name.startswith("pack-") and name.endswith(".pack"):   filename = os.path.join(self.pack_dir, name) @@ -348,6 +355,14 @@
  pack_files.sort(reverse=True)   suffix_len = len(".pack")   return [Pack(f[:-suffix_len]) for _, f in pack_files] + + def _pack_cache_stale(self): + try: + return os.stat(self.pack_dir).st_mtime > self._pack_cache_time + except OSError, e: + if e.errno == errno.ENOENT: + return True + raise     def _get_shafile_path(self, sha):   dir = sha[:2]
Change 1 of 1 Show Entire File dulwich/​server.py Stacked
 
509
510
511
512
513
514
515
516
517
518
519
520
 
509
510
511
 
 
 
 
 
 
512
513
514
@@ -509,12 +509,6 @@
  self.stateless_rpc = stateless_rpc   self.advertise_refs = advertise_refs   - def __init__(self, backend, read, write, - stateless_rpc=False, advertise_refs=False): - Handler.__init__(self, backend, read, write) - self._stateless_rpc = stateless_rpc - self._advertise_refs = advertise_refs -   def capabilities(self):   return ("report-status", "delete-refs")  
Change 1 of 1 Show Entire File dulwich/​tests/​compat/​server_utils.py Stacked
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
@@ -1,0 +1,169 @@
+# server_utils.py -- Git server compatibility utilities +# Copyright (C) 2010 Google, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License or (at your option) any later version of +# the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +"""Utilities for testing git server compatibility.""" + + +import select +import socket +import threading + +from dulwich.tests.utils import ( + tear_down_repo, + ) +from utils import ( + import_repo, + run_git, + ) + + +class ServerTests(object): + """Base tests for testing servers. + + Does not inherit from TestCase so tests are not automatically run. + """ + + def setUp(self): + self._old_repo = import_repo('server_old.export') + self._new_repo = import_repo('server_new.export') + self._server = None + + def tearDown(self): + if self._server is not None: + self._server.shutdown() + self._server = None + tear_down_repo(self._old_repo) + tear_down_repo(self._new_repo) + + def assertReposEqual(self, repo1, repo2): + self.assertEqual(repo1.get_refs(), repo2.get_refs()) + self.assertEqual(set(repo1.object_store), set(repo2.object_store)) + + def assertReposNotEqual(self, repo1, repo2): + refs1 = repo1.get_refs() + objs1 = set(repo1.object_store) + refs2 = repo2.get_refs() + objs2 = set(repo2.object_store) + + self.assertFalse(refs1 == refs2 and objs1 == objs2) + + def test_push_to_dulwich(self): + self.assertReposNotEqual(self._old_repo, self._new_repo) + port = self._start_server(self._old_repo) + + all_branches = ['master', 'branch'] + branch_args = ['%s:%s' % (b, b) for b in all_branches] + url = '%s://localhost:%s/' % (self.protocol, port) + returncode, _ = run_git(['push', url] + branch_args, + cwd=self._new_repo.path) + self.assertEqual(0, returncode) + self.assertReposEqual(self._old_repo, self._new_repo) + + def test_fetch_from_dulwich(self): + self.assertReposNotEqual(self._old_repo, self._new_repo) + port = self._start_server(self._new_repo) + + all_branches = ['master', 'branch'] + branch_args = ['%s:%s' % (b, b) for b in all_branches] + url = '%s://localhost:%s/' % (self.protocol, port) + returncode, _ = run_git(['fetch', url] + branch_args, + cwd=self._old_repo.path) + # flush the pack cache so any new packs are picked up + self._old_repo.object_store._pack_cache = None + self.assertEqual(0, returncode) + self.assertReposEqual(self._old_repo, self._new_repo) + + +class ShutdownServerMixIn: + """Mixin that allows serve_forever to be shut down. + + The methods in this mixin are backported from SocketServer.py in the Python + 2.6.4 standard library. The mixin is unnecessary in 2.6 and later, when + BaseServer supports the shutdown method directly. + """ + + def __init__(self): + self.__is_shut_down = threading.Event() + self.__serving = False + + def serve_forever(self, poll_interval=0.5): + """Handle one request at a time until shutdown. + + Polls for shutdown every poll_interval seconds. Ignores + self.timeout. If you need to do periodic tasks, do them in + another thread. + """ + self.__serving = True + self.__is_shut_down.clear() + while self.__serving: + # XXX: Consider using another file descriptor or + # connecting to the socket to wake this up instead of + # polling. Polling reduces our responsiveness to a + # shutdown request and wastes cpu at all other times. + r, w, e = select.select([self], [], [], poll_interval) + if r: + self._handle_request_noblock() + self.__is_shut_down.set() + + serve = serve_forever # override alias from TCPGitServer + + def shutdown(self): + """Stops the serve_forever loop. + + Blocks until the loop has finished. This must be called while + serve_forever() is running in another thread, or it will deadlock. + """ + self.__serving = False + self.__is_shut_down.wait() + + def handle_request(self): + """Handle one request, possibly blocking. + + Respects self.timeout. + """ + # Support people who used socket.settimeout() to escape + # handle_request before self.timeout was available. + timeout = self.socket.gettimeout() + if timeout is None: + timeout = self.timeout + elif self.timeout is not None: + timeout = min(timeout, self.timeout) + fd_sets = select.select([self], [], [], timeout) + if not fd_sets[0]: + self.handle_timeout() + return + self._handle_request_noblock() + + def _handle_request_noblock(self): + """Handle one request, without blocking. + + I assume that select.select has returned that the socket is + readable before this function was called, so there should be + no risk of blocking in get_request(). + """ + try: + request, client_address = self.get_request() + except socket.error: + return + if self.verify_request(request, client_address): + try: + self.process_request(request, client_address) + except: + self.handle_error(request, client_address) + self.close_request(request)
Change 1 of 1 Show Entire File dulwich/​tests/​compat/​test_pack.py Stacked
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
@@ -1,0 +1,74 @@
+# test_pack.py -- Compatibilty tests for git packs. +# Copyright (C) 2010 Google, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License or (at your option) any later version of +# the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +"""Compatibilty tests for git packs.""" + + +import binascii +import os +import shutil +import tempfile + +from dulwich.pack import ( + Pack, + write_pack, + ) +from dulwich.tests.test_pack import ( + pack1_sha, + PackTests, + ) +from utils import ( + require_git_version, + run_git, + ) + + +class TestPack(PackTests): + """Compatibility tests for reading and writing pack files.""" + + def setUp(self): + require_git_version((1, 5, 0)) + PackTests.setUp(self) + self._tempdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._tempdir) + PackTests.tearDown(self) + + def test_copy(self): + origpack = self.get_pack(pack1_sha) + self.assertEquals(True, origpack.index.check()) + pack_path = os.path.join(self._tempdir, "Elch") + write_pack(pack_path, [(x, "") for x in origpack.iterobjects()], + len(origpack)) + + returncode, output = run_git(['verify-pack', '-v', pack_path], + capture_stdout=True) + self.assertEquals(0, returncode) + + pack_shas = set() + for line in output.splitlines(): + sha = line[:40] + try: + binascii.unhexlify(sha) + except TypeError: + continue # non-sha line + pack_shas.add(sha) + orig_shas = set(o.id for o in origpack.iterobjects()) + self.assertEquals(orig_shas, pack_shas)
Change 1 of 1 Show Entire File dulwich/​tests/​compat/​test_server.py Stacked
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
@@ -1,0 +1,77 @@
+# test_server.py -- Compatibilty tests for git server. +# Copyright (C) 2010 Google, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License or (at your option) any later version of +# the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +"""Compatibilty tests between Dulwich and the cgit server. + +Warning: these tests should be fairly stable, but when writing/debugging new +tests, deadlocks may freeze the test process such that it cannot be Ctrl-C'ed. +On *nix, you can kill the tests with Ctrl-Z, "kill %". +""" + +import threading +import unittest + +import nose + +from dulwich import server +from server_utils import ( + ServerTests, + ShutdownServerMixIn, + ) +from utils import ( + CompatTestCase, + ) + + +if getattr(server.TCPGitServer, 'shutdown', None): + TCPGitServer = server.TCPGitServer +else: + class TCPGitServer(ShutdownServerMixIn, server.TCPGitServer): + """Subclass of TCPGitServer that can be shut down.""" + + def __init__(self, *args, **kwargs): + # BaseServer is old-style so we have to call both __init__s + ShutdownServerMixIn.__init__(self) + server.TCPGitServer.__init__(self, *args, **kwargs) + + serve = ShutdownServerMixIn.serve_forever + + +class GitServerTestCase(ServerTests, CompatTestCase): + """Tests for client/server compatibility.""" + + protocol = 'git' + + def setUp(self): + ServerTests.setUp(self) + CompatTestCase.setUp(self) + + def tearDown(self): + ServerTests.tearDown(self) + CompatTestCase.tearDown(self) + + def _start_server(self, repo): + dul_server = TCPGitServer(server.GitBackend(repo), 'localhost', 0) + threading.Thread(target=dul_server.serve).start() + self._server = dul_server + _, port = self._server.socket.getsockname() + return port + + def test_push_to_dulwich(self): + raise nose.SkipTest('Skipping push test due to known deadlock bug.')
Change 1 of 1 Show Entire File dulwich/​tests/​compat/​test_web.py Stacked
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
@@ -1,0 +1,127 @@
+# test_web.py -- Compatibilty tests for the git web server. +# Copyright (C) 2010 Google, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License or (at your option) any later version of +# the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +"""Compatibilty tests between Dulwich and the cgit HTTP server. + +Warning: these tests should be fairly stable, but when writing/debugging new +tests, deadlocks may freeze the test process such that it cannot be Ctrl-C'ed. +On *nix, you can kill the tests with Ctrl-Z, "kill %". +""" + +import sys +import threading +import unittest +from wsgiref import simple_server + +import nose + +from dulwich.repo import ( + Repo, + ) +from dulwich.server import ( + GitBackend, + ) +from dulwich.web import ( + HTTPGitApplication, + ) + +from dulwich.tests.utils import ( + open_repo, + tear_down_repo, + ) +from server_utils import ( + ServerTests, + ShutdownServerMixIn, + ) +from utils import ( + CompatTestCase, + ) + + +if getattr(simple_server.WSGIServer, 'shutdown', None): + WSGIServer = simple_server.WSGIServer +else: + class WSGIServer(ShutdownServerMixIn, simple_server.WSGIServer): + """Subclass of WSGIServer that can be shut down.""" + + def __init__(self, *args, **kwargs): + # BaseServer is old-style so we have to call both __init__s + ShutdownServerMixIn.__init__(self) + simple_server.WSGIServer.__init__(self, *args, **kwargs) + + serve = ShutdownServerMixIn.serve_forever + + +class WebTests(ServerTests): + """Base tests for web server tests. + + Contains utility and setUp/tearDown methods, but does non inherit from + TestCase so tests are not automatically run. + """ + + protocol = 'http' + + def _start_server(self, repo): + app = self._make_app(GitBackend(repo)) + dul_server = simple_server.make_server('localhost', 0, app, + server_class=WSGIServer) + threading.Thread(target=dul_server.serve_forever).start() + self._server = dul_server + _, port = dul_server.socket.getsockname() + return port + + +class SmartWebTestCase(WebTests, CompatTestCase): + """Test cases for smart HTTP server.""" + + min_git_version = (1, 6, 6) + + def setUp(self): + WebTests.setUp(self) + CompatTestCase.setUp(self) + + def tearDown(self): + WebTests.tearDown(self) + CompatTestCase.tearDown(self) + + def _make_app(self, backend): + return HTTPGitApplication(backend) + + def test_push_to_dulwich(self): + # TODO(dborowitz): enable after merging thin pack fixes. + raise nose.SkipTest('Skipping push test due to known pack bug.') + + +class DumbWebTestCase(WebTests, CompatTestCase): + """Test cases for dumb HTTP server.""" + + def setUp(self): + WebTests.setUp(self) + CompatTestCase.setUp(self) + + def tearDown(self): + WebTests.tearDown(self) + CompatTestCase.tearDown(self) + + def _make_app(self, backend): + return HTTPGitApplication(backend, dumb=True) + + def test_push_to_dulwich(self): + # Note: remove this if dumb pushing is supported + raise nose.SkipTest('Dumb web pushing not supported.')
Change 1 of 1 Show Entire File dulwich/​tests/​compat/​utils.py Stacked
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
@@ -1,0 +1,143 @@
+# utils.py -- Git compatibility utilities +# Copyright (C) 2010 Google, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License or (at your option) any later version of +# the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +"""Utilities for interacting with cgit.""" + +import os +import subprocess +import tempfile +import unittest + +import nose + +from dulwich.repo import Repo +from dulwich.tests.utils import open_repo + + +_DEFAULT_GIT = 'git' + + +def git_version(git_path=_DEFAULT_GIT): + """Attempt to determine the version of git currently installed. + + :param git_path: Path to the git executable; defaults to the version in + the system path. + :return: A tuple of ints of the form (major, minor, point), or None if no + git installation was found. + """ + try: + _, output = run_git(['--version'], git_path=git_path, + capture_stdout=True) + except OSError: + return None + version_prefix = 'git version ' + if not output.startswith(version_prefix): + return None + output = output[len(version_prefix):] + nums = output.split('.') + if len(nums) == 2: + nums.add('0') + else: + nums = nums[:3] + try: + return tuple(int(x) for x in nums) + except ValueError: + return None + + +def require_git_version(required_version, git_path=_DEFAULT_GIT): + """Require git version >= version, or skip the calling test.""" + found_version = git_version(git_path=git_path) + if found_version < required_version: + required_version = '.'.join(map(str, required_version)) + found_version = '.'.join(map(str, found_version)) + raise nose.SkipTest('Test requires git >= %s, found %s' % + (required_version, found_version)) + + +def run_git(args, git_path=_DEFAULT_GIT, input=None, capture_stdout=False, + **popen_kwargs): + """Run a git command. + + Input is piped from the input parameter and output is sent to the standard + streams, unless capture_stdout is set. + + :param args: A list of args to the git command. + :param git_path: Path to to the git executable. + :param input: Input data to be sent to stdin. + :param capture_stdout: Whether to capture and return stdout. + :param popen_kwargs: Additional kwargs for subprocess.Popen; + stdin/stdout args are ignored. + :return: A tuple of (returncode, stdout contents). If capture_stdout is + False, None will be returned as stdout contents. + :raise OSError: if the git executable was not found. + """ + args = [git_path] + args + popen_kwargs['stdin'] = subprocess.PIPE + if capture_stdout: + popen_kwargs['stdout'] = subprocess.PIPE + else: + popen_kwargs.pop('stdout', None) + p = subprocess.Popen(args, **popen_kwargs) + stdout, stderr = p.communicate(input=input) + return (p.returncode, stdout) + + +def run_git_or_fail(args, git_path=_DEFAULT_GIT, input=None, **popen_kwargs): + """Run a git command, capture stdout/stderr, and fail if git fails.""" + popen_kwargs['stderr'] = subprocess.STDOUT + returncode, stdout = run_git(args, git_path=git_path, input=input, + capture_stdout=True, **popen_kwargs) + assert returncode == 0 + return stdout + + +def import_repo(name): + """Import a repo from a fast-export file in a temporary directory. + + These are used rather than binary repos for compat tests because they are + more compact an human-editable, and we already depend on git. + + :param name: The name of the repository export file, relative to + dulwich/tests/data/repos + :returns: An initialized Repo object that lives in a temporary directory. + """ + temp_dir = tempfile.mkdtemp() + export_path = os.path.join(os.path.dirname(__file__), os.pardir, 'data', + 'repos', name) + temp_repo_dir = os.path.join(temp_dir, name) + export_file = open(export_path, 'rb') + run_git_or_fail(['init', '--bare', temp_repo_dir]) + run_git_or_fail(['fast-import'], input=export_file.read(), + cwd=temp_repo_dir) + export_file.close() + return Repo(temp_repo_dir) + + +class CompatTestCase(unittest.TestCase): + """Test case that requires git for compatibility checks. + + Subclasses can change the git version required by overriding + min_git_version. + """ + + min_git_version = (1, 5, 0) + + def setUp(self): + require_git_version(self.min_git_version)
Change 1 of 1 Show Entire File dulwich/​tests/​data/​repos/​server_new.export Stacked
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
@@ -1,0 +1,99 @@
+blob +mark :1 +data 13 +foo contents + +reset refs/heads/master +commit refs/heads/master +mark :2 +author Dave Borowitz <dborowitz@google.com> 1265755064 -0800 +committer Dave Borowitz <dborowitz@google.com> 1265755064 -0800 +data 16 +initial checkin +M 100644 :1 foo + +blob +mark :3 +data 13 +baz contents + +blob +mark :4 +data 21 +updated foo contents + +commit refs/heads/master +mark :5 +author Dave Borowitz <dborowitz@google.com> 1265755140 -0800 +committer Dave Borowitz <dborowitz@google.com> 1265755140 -0800 +data 15 +master checkin +from :2 +M 100644 :3 baz +M 100644 :4 foo + +blob +mark :6 +data 24 +updated foo contents v2 + +commit refs/heads/master +mark :7 +author Dave Borowitz <dborowitz@google.com> 1265755287 -0800 +committer Dave Borowitz <dborowitz@google.com> 1265755287 -0800 +data 17 +master checkin 2 +from :5 +M 100644 :6 foo + +blob +mark :8 +data 24 +updated foo contents v3 + +commit refs/heads/master +mark :9 +author Dave Borowitz <dborowitz@google.com> 1265755295 -0800 +committer Dave Borowitz <dborowitz@google.com> 1265755295 -0800 +data 17 +master checkin 3 +from :7 +M 100644 :8 foo + +blob +mark :10 +data 22 +branched bar contents + +blob +mark :11 +data 22 +branched foo contents + +commit refs/heads/branch +mark :12 +author Dave Borowitz <dborowitz@google.com> 1265755111 -0800 +committer Dave Borowitz <dborowitz@google.com> 1265755111 -0800 +data 15 +branch checkin +from :2 +M 100644 :10 bar +M 100644 :11 foo + +blob +mark :13 +data 25 +branched bar contents v2 + +commit refs/heads/branch +mark :14 +author Dave Borowitz <dborowitz@google.com> 1265755319 -0800 +committer Dave Borowitz <dborowitz@google.com> 1265755319 -0800 +data 17 +branch checkin 2 +from :12 +M 100644 :13 bar + +reset refs/heads/master +from :9 +
Change 1 of 1 Show Entire File dulwich/​tests/​data/​repos/​server_old.export Stacked
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
@@ -1,0 +1,57 @@
+blob +mark :1 +data 13 +foo contents + +reset refs/heads/master +commit refs/heads/master +mark :2 +author Dave Borowitz <dborowitz@google.com> 1265755064 -0800 +committer Dave Borowitz <dborowitz@google.com> 1265755064 -0800 +data 16 +initial checkin +M 100644 :1 foo + +blob +mark :3 +data 22 +branched bar contents + +blob +mark :4 +data 22 +branched foo contents + +commit refs/heads/branch +mark :5 +author Dave Borowitz <dborowitz@google.com> 1265755111 -0800 +committer Dave Borowitz <dborowitz@google.com> 1265755111 -0800 +data 15 +branch checkin +from :2 +M 100644 :3 bar +M 100644 :4 foo + +blob +mark :6 +data 13 +baz contents + +blob +mark :7 +data 21 +updated foo contents + +commit refs/heads/master +mark :8 +author Dave Borowitz <dborowitz@google.com> 1265755140 -0800 +committer Dave Borowitz <dborowitz@google.com> 1265755140 -0800 +data 15 +master checkin +from :2 +M 100644 :6 baz +M 100644 :7 foo + +reset refs/heads/master +from :8 +
 
35
36
37
 
 
 
 
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
82
83
84
85
 
86
87
88
 
219
220
221
222
223
224
225
 
268
269
270
271
272
273
274
 
35
36
37
38
39
40
41
42
43
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
45
46
 
62
63
64
 
65
66
67
68
 
199
200
201
 
202
203
204
 
247
248
249
 
250
251
252
@@ -35,32 +35,12 @@
  write_packed_refs,   _split_ref_line,   ) +from dulwich.tests.utils import ( + open_repo, + tear_down_repo, + )    missing_sha = 'b91fa4d900e17e99b433218e988c4eb4a3e9a097' - - -def open_repo(name): - """Open a copy of a repo in a temporary directory. - - Use this function for accessing repos in dulwich/tests/data/repos to avoid - accidentally or intentionally modifying those repos in place. Use - tear_down_repo to delete any temp files created. - - :param name: The name of the repository, relative to - dulwich/tests/data/repos - :returns: An initialized Repo object that lives in a temporary directory. - """ - temp_dir = tempfile.mkdtemp() - repo_dir = os.path.join(os.path.dirname(__file__), 'data', 'repos', name) - temp_repo_dir = os.path.join(temp_dir, name) - shutil.copytree(repo_dir, temp_repo_dir, symlinks=True) - return Repo(temp_repo_dir) - -def tear_down_repo(repo): - """Tear down a test repository.""" - temp_dir = os.path.dirname(repo.path.rstrip(os.sep)) - shutil.rmtree(temp_dir) -      class CreateRepositoryTests(unittest.TestCase): @@ -82,7 +62,7 @@
  def tearDown(self):   if self._repo is not None:   tear_down_repo(self._repo) - +   def test_simple_props(self):   r = self._repo = open_repo('a.git')   self.assertEqual(r.controldir(), r.path) @@ -219,7 +199,6 @@
 FOURS = "4" * 40    class PackedRefsFileTests(unittest.TestCase): -   def test_split_ref_line_errors(self):   self.assertRaises(errors.PackedRefsException, _split_ref_line,   'singlefield') @@ -268,7 +247,6 @@
     class RefsContainerTests(unittest.TestCase): -   def setUp(self):   self._repo = open_repo('refs.git')   self._refs = self._repo.refs
Change 1 of 1 Show Entire File dulwich/​tests/​utils.py Stacked
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
@@ -1,0 +1,51 @@
+# utils.py -- Test utilities for Dulwich. +# Copyright (C) 2010 Google, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License or (at your option) any later version of +# the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +"""Utility functions common to Dulwich tests.""" + + +import os +import shutil +import tempfile + +from dulwich.repo import Repo + + +def open_repo(name): + """Open a copy of a repo in a temporary directory. + + Use this function for accessing repos in dulwich/tests/data/repos to avoid + accidentally or intentionally modifying those repos in place. Use + tear_down_repo to delete any temp files created. + + :param name: The name of the repository, relative to + dulwich/tests/data/repos + :returns: An initialized Repo object that lives in a temporary directory. + """ + temp_dir = tempfile.mkdtemp() + repo_dir = os.path.join(os.path.dirname(__file__), 'data', 'repos', name) + temp_repo_dir = os.path.join(temp_dir, name) + shutil.copytree(repo_dir, temp_repo_dir, symlinks=True) + return Repo(temp_repo_dir) + + +def tear_down_repo(repo): + """Tear down a test repository.""" + temp_dir = os.path.dirname(repo.path.rstrip(os.sep)) + shutil.rmtree(temp_dir)
Change 1 of 4 Show Entire File dulwich/​web.py Stacked
 
104
105
106
107
 
108
109
110
111
112
113
114
115
116
117
 
 
 
 
 
118
119
120
 
121
122
123
 
124
125
126
 
190
191
192
193
 
194
195
 
196
197
198
 
220
221
222
223
 
224
 
225
226
227
 
290
291
292
293
 
294
 
295
296
297
298
299
 
300
301
302
 
104
105
106
 
107
108
109
110
111
112
 
 
 
 
 
113
114
115
116
117
118
119
 
120
121
122
 
123
124
125
126
 
190
191
192
 
193
194
 
195
196
197
198
 
220
221
222
 
223
224
225
226
227
228
 
291
292
293
 
294
295
296
297
298
299
300
 
301
302
303
304
@@ -104,23 +104,23 @@
 def get_pack_file(req, backend, mat):   req.cache_forever()   return send_file(req, backend.repo.get_named_file(mat.group()), - 'application/x-git-packed-objects', False) + 'application/x-git-packed-objects')      def get_idx_file(req, backend, mat):   req.cache_forever()   return send_file(req, backend.repo.get_named_file(mat.group()), - 'application/x-git-packed-objects-toc', False) - - -services = {'git-upload-pack': UploadPackHandler, - 'git-receive-pack': ReceivePackHandler} + 'application/x-git-packed-objects-toc') + + +default_services = {'git-upload-pack': UploadPackHandler, + 'git-receive-pack': ReceivePackHandler}  def get_info_refs(req, backend, mat, services=None):   if services is None: - services = services + services = default_services   params = cgi.parse_qs(req.environ['QUERY_STRING'])   service = params.get('service', [None])[0] - if service: + if service and not req.dumb:   handler_cls = services.get(service, None)   if handler_cls is None:   yield req.forbidden('Unsupported service %s' % service) @@ -190,9 +190,9 @@
    # TODO: support more methods as necessary   -def handle_service_request(req, backend, mat, services=services): +def handle_service_request(req, backend, mat, services=None):   if services is None: - services = services + services = default_services   service = mat.group().lstrip('/')   handler_cls = services.get(service, None)   if handler_cls is None: @@ -220,8 +220,9 @@
  :ivar environ: the WSGI environment for the request.   """   - def __init__(self, environ, start_response): + def __init__(self, environ, start_response, dumb=False):   self.environ = environ + self.dumb = dumb   self._start_response = start_response   self._cache_headers = []   self._headers = [] @@ -290,13 +291,14 @@
  ('POST', re.compile('/git-receive-pack$')): handle_service_request,   }   - def __init__(self, backend): + def __init__(self, backend, dumb=False):   self.backend = backend + self.dumb = dumb     def __call__(self, environ, start_response):   path = environ['PATH_INFO']   method = environ['REQUEST_METHOD'] - req = HTTPGitRequest(environ, start_response) + req = HTTPGitRequest(environ, start_response, self.dumb)   # environ['QUERY_STRING'] has qs args   handler = None   for smethod, spath in self.services.iterkeys():