Skip to content

Commit 9748ee3

Browse files
ajnavarroajnavarro
committed
Perf: Traverse commit history instead of reading the packfile
To avoid show orphan commits and improve performance on commits table, instead of read all the objects from each repository and discard commits, we can execute from go-git a git log --all command and traverse commit history, avoiding read a lot of data from disk. Signed-off-by: Antonio Jesus Navarro Perez <[email protected]>
1 parent 73c1730 commit 9748ee3

13 files changed

+117
-75
lines changed

‎commit_blobs.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"io"
66

7+
git "gopkg.in/src-d/go-git.v4"
78
"gopkg.in/src-d/go-git.v4/plumbing"
89
"gopkg.in/src-d/go-git.v4/plumbing/object"
910
"gopkg.in/src-d/go-mysql-server.v0/sql"
@@ -219,7 +220,9 @@ func (i *commitBlobsRowIter) init() error {
219220
if len(i.commits) > 0 {
220221
i.iter, err = NewCommitsByHashIter(i.repo, i.commits)
221222
} else {
222-
i.iter, err = i.repo.CommitObjects()
223+
i.iter, err = i.repo.Log(&git.LogOptions{
224+
All: true,
225+
})
223226
}
224227

225228
return err

‎commit_files.go

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"io"
66

77
"github.com/sirupsen/logrus"
8+
git "gopkg.in/src-d/go-git.v4"
89
"gopkg.in/src-d/go-git.v4/plumbing"
910
"gopkg.in/src-d/go-git.v4/plumbing/filemode"
1011
"gopkg.in/src-d/go-git.v4/plumbing/object"
@@ -196,7 +197,10 @@ func (i *commitFilesRowIter) init() error {
196197
if len(i.commitHashes) > 0 {
197198
i.commits, err = NewCommitsByHashIter(i.repo, i.commitHashes)
198199
} else {
199-
i.commits, err = i.repo.CommitObjects()
200+
i.commits, err = i.repo.
201+
Log(&git.LogOptions{
202+
All: true,
203+
})
200204
}
201205

202206
return err
@@ -433,7 +437,10 @@ func newCommitFilesKeyValueIter(
433437
return nil, err
434438
}
435439

436-
commits, err := repo.CommitObjects()
440+
commits, err := repo.
441+
Log(&git.LogOptions{
442+
All: true,
443+
})
437444
if err != nil {
438445
return nil, err
439446
}

‎commit_files_test.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"testing"
66

77
"github.com/stretchr/testify/require"
8+
git "gopkg.in/src-d/go-git.v4"
89
"gopkg.in/src-d/go-git.v4/plumbing"
910
"gopkg.in/src-d/go-mysql-server.v0/sql"
1011
"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
@@ -35,7 +36,9 @@ func TestCommitFilesTableRowIter(t *testing.T) {
3536

3637
require.NoError(err)
3738

38-
commits, err := repo.CommitObjects()
39+
commits, err := repo.Log(&git.LogOptions{
40+
All: true,
41+
})
3942
require.NoError(err)
4043

4144
for {

‎commit_trees.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"io"
66

7+
git "gopkg.in/src-d/go-git.v4"
78
"gopkg.in/src-d/go-git.v4/plumbing"
89

910
"gopkg.in/src-d/go-git.v4/plumbing/filemode"
@@ -215,7 +216,9 @@ func (i *commitTreesRowIter) init() error {
215216
if len(i.commitHashes) > 0 {
216217
i.commits, err = NewCommitsByHashIter(i.repo, i.commitHashes)
217218
} else {
218-
i.commits, err = i.repo.CommitObjects()
219+
i.commits, err = i.repo.Log(&git.LogOptions{
220+
All: true,
221+
})
219222
}
220223

221224
return err

‎commit_trees_test.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,17 @@ func TestCommitTreesRowIter(t *testing.T) {
2323
}
2424

2525
expected := []sql.Row{
26-
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "dbd3641b371024f44d0e469a9c8f5457b0660de1"},
27-
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "a39771a7651f97faf5c72e08224d857fc35133db"},
28-
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
29-
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
30-
3126
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a8d315b2b1c615d43042c3a62402b8a54288cf5c"},
3227
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a39771a7651f97faf5c72e08224d857fc35133db"},
3328
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
3429
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
3530
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "cf4aa3b38974fb7d81f367c0830f7d78d65ab86b"},
3631

32+
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "dbd3641b371024f44d0e469a9c8f5457b0660de1"},
33+
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "a39771a7651f97faf5c72e08224d857fc35133db"},
34+
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
35+
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
36+
3737
{"918c48b83bd081e863dbe1b80f8998f058cd8294", "fb72698cab7617ac416264415f13224dfd7a165e"},
3838
{"918c48b83bd081e863dbe1b80f8998f058cd8294", "a39771a7651f97faf5c72e08224d857fc35133db"},
3939
{"918c48b83bd081e863dbe1b80f8998f058cd8294", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
@@ -44,13 +44,13 @@ func TestCommitTreesRowIter(t *testing.T) {
4444

4545
{"1669dce138d9b841a518c64b10914d88f5e488ea", "eba74343e2f15d62adedfd8c883ee0262b5c8021"},
4646

47-
{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},
48-
49-
{"b8e471f58bcbca63b07bda20e428190409c2db47", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},
50-
5147
{"35e85108805c84807bc66a02d91535e1e24b38b9", "8dcef98b1d52143e1e2dbc458ffe38f925786bf2"},
5248

5349
{"b029517f6300c2da0f4b651b8642506cd6aaf45d", "aa9b383c260e1d05fbbf6b30a02914555e20c725"},
50+
51+
{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},
52+
53+
{"b8e471f58bcbca63b07bda20e428190409c2db47", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},
5454
}
5555

5656
require.Equal(expected, rows)
@@ -90,8 +90,8 @@ func TestCommitTreesPushdown(t *testing.T) {
9090
),
9191
},
9292
[]sql.Row{
93-
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
9493
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
94+
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
9595
{"918c48b83bd081e863dbe1b80f8998f058cd8294", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
9696
},
9797
},

‎commits.go

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package gitbase
33
import (
44
"io"
55

6+
git "gopkg.in/src-d/go-git.v4"
67
"gopkg.in/src-d/go-mysql-server.v0/sql"
78

89
"gopkg.in/src-d/go-git.v4/plumbing"
@@ -161,7 +162,10 @@ func (i *commitIter) init() error {
161162
if len(i.hashes) > 0 {
162163
i.iter, err = NewCommitsByHashIter(i.repo, i.hashes)
163164
} else {
164-
i.iter, err = i.repo.CommitObjects()
165+
i.iter, err =
166+
i.repo.Log(&git.LogOptions{
167+
All: true,
168+
})
165169
}
166170

167171
return err
@@ -245,7 +249,10 @@ func NewCommitsByHashIter(
245249
var commitIter object.CommitIter
246250
var err error
247251
if len(hashes) == 0 {
248-
commitIter, err = repo.CommitObjects()
252+
commitIter, err =
253+
repo.Log(&git.LogOptions{
254+
All: true,
255+
})
249256
if err != nil {
250257
return nil, err
251258
}
@@ -333,7 +340,10 @@ func newCommitsKeyValueIter(
333340
return nil, err
334341
}
335342

336-
commits, err := repo.CommitObjects()
343+
commits, err :=
344+
repo.Log(&git.LogOptions{
345+
All: true,
346+
})
337347
if err != nil {
338348
return nil, err
339349
}

‎commits_test.go

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -101,14 +101,14 @@ func TestCommitsParents(t *testing.T) {
101101
}{
102102
{
103103
name: "test commits parents 1",
104-
hash: "e8d3ffab552895c19b9fcf7aa264d277cde33881",
104+
hash: "6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
105105
parents: []string{
106106
"918c48b83bd081e863dbe1b80f8998f058cd8294",
107107
},
108108
},
109109
{
110110
name: "test commits parents 2",
111-
hash: "6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
111+
hash: "e8d3ffab552895c19b9fcf7aa264d277cde33881",
112112
parents: []string{
113113
"918c48b83bd081e863dbe1b80f8998f058cd8294",
114114
},
@@ -137,30 +137,30 @@ func TestCommitsParents(t *testing.T) {
137137
},
138138
{
139139
name: "test commits parents 6",
140-
hash: "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
140+
hash: "35e85108805c84807bc66a02d91535e1e24b38b9",
141141
parents: []string{
142142
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
143-
"b8e471f58bcbca63b07bda20e428190409c2db47",
144143
},
145144
},
146145
{
147-
name: "test commits parents 7",
148-
hash: "b8e471f58bcbca63b07bda20e428190409c2db47",
149-
parents: []string{
150-
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
151-
},
146+
name: "test commits parents 7",
147+
hash: "b029517f6300c2da0f4b651b8642506cd6aaf45d",
148+
parents: []string{},
152149
},
153150
{
154151
name: "test commits parents 8",
155-
hash: "35e85108805c84807bc66a02d91535e1e24b38b9",
152+
hash: "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
156153
parents: []string{
157154
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
155+
"b8e471f58bcbca63b07bda20e428190409c2db47",
158156
},
159157
},
160158
{
161-
name: "test commits parents 9",
162-
hash: "b029517f6300c2da0f4b651b8642506cd6aaf45d",
163-
parents: []string{},
159+
name: "test commits parents 9",
160+
hash: "b8e471f58bcbca63b07bda20e428190409c2db47",
161+
parents: []string{
162+
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
163+
},
164164
},
165165
}
166166

@@ -188,65 +188,65 @@ func TestCommitsIndexKeyValueIter(t *testing.T) {
188188
assertEncodeKey(t, &packOffsetIndexKey{
189189
Repository: path,
190190
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
191-
Offset: 682,
191+
Offset: 186,
192192
}),
193193
[]interface{}{
194-
"1669dce138d9b841a518c64b10914d88f5e488ea",
194+
"6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
195195
"[email protected]",
196196
},
197197
},
198198
{
199199
assertEncodeKey(t, &packOffsetIndexKey{
200200
Repository: path,
201201
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
202-
Offset: 1292,
202+
Offset: 12,
203203
}),
204204
[]interface{}{
205-
"35e85108805c84807bc66a02d91535e1e24b38b9",
205+
"e8d3ffab552895c19b9fcf7aa264d277cde33881",
206206
"[email protected]",
207207
},
208208
},
209209
{
210210
assertEncodeKey(t, &packOffsetIndexKey{
211211
Repository: path,
212212
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
213-
Offset: 186,
213+
Offset: 353,
214214
}),
215215
[]interface{}{
216-
"6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
216+
"918c48b83bd081e863dbe1b80f8998f058cd8294",
217217
"[email protected]",
218218
},
219219
},
220220
{
221221
assertEncodeKey(t, &packOffsetIndexKey{
222222
Repository: path,
223223
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
224-
Offset: 353,
224+
Offset: 516,
225225
}),
226226
[]interface{}{
227-
"918c48b83bd081e863dbe1b80f8998f058cd8294",
227+
"af2d6a6954d532f8ffb47615169c8fdf9d383a1a",
228228
"[email protected]",
229229
},
230230
},
231231
{
232232
assertEncodeKey(t, &packOffsetIndexKey{
233233
Repository: path,
234234
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
235-
Offset: 905,
235+
Offset: 682,
236236
}),
237237
[]interface{}{
238-
"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
238+
"1669dce138d9b841a518c64b10914d88f5e488ea",
239239
"[email protected]",
240240
},
241241
},
242242
{
243243
assertEncodeKey(t, &packOffsetIndexKey{
244244
Repository: path,
245245
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
246-
Offset: 516,
246+
Offset: 1292,
247247
}),
248248
[]interface{}{
249-
"af2d6a6954d532f8ffb47615169c8fdf9d383a1a",
249+
"35e85108805c84807bc66a02d91535e1e24b38b9",
250250
"[email protected]",
251251
},
252252
},
@@ -265,22 +265,22 @@ func TestCommitsIndexKeyValueIter(t *testing.T) {
265265
assertEncodeKey(t, &packOffsetIndexKey{
266266
Repository: path,
267267
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
268-
Offset: 1130,
268+
Offset: 905,
269269
}),
270270
[]interface{}{
271-
"b8e471f58bcbca63b07bda20e428190409c2db47",
272-
"[email protected]",
271+
"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
272+
"[email protected]",
273273
},
274274
},
275275
{
276276
assertEncodeKey(t, &packOffsetIndexKey{
277277
Repository: path,
278278
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
279-
Offset: 12,
279+
Offset: 1130,
280280
}),
281281
[]interface{}{
282-
"e8d3ffab552895c19b9fcf7aa264d277cde33881",
283-
"[email protected]",
282+
"b8e471f58bcbca63b07bda20e428190409c2db47",
283+
"[email protected]",
284284
},
285285
},
286286
}

‎files.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"io"
66

7+
git "gopkg.in/src-d/go-git.v4"
78
"gopkg.in/src-d/go-git.v4/plumbing"
89
"gopkg.in/src-d/go-git.v4/plumbing/filemode"
910
"gopkg.in/src-d/go-git.v4/plumbing/object"
@@ -192,7 +193,9 @@ type filesRowIter struct {
192193
func (i *filesRowIter) init() error {
193194
var err error
194195
i.seen = make(map[plumbing.Hash]struct{})
195-
i.commits, err = i.repo.CommitObjects()
196+
i.commits, err = i.repo.Log(&git.LogOptions{
197+
All: true,
198+
})
196199
return err
197200
}
198201

@@ -411,7 +414,9 @@ func newFilesKeyValueIter(pool *RepositoryPool, repo *Repository, columns []stri
411414
return nil, err
412415
}
413416

414-
commits, err := repo.CommitObjects()
417+
commits, err := repo.Log(&git.LogOptions{
418+
All: true,
419+
})
415420
if err != nil {
416421
return nil, err
417422
}

0 commit comments

Comments
 (0)