Skip to content

test(e2e): make plugin e2e tests pass on a fresh database #9224

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chenwei791129 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

test(e2e): make plugin e2e tests pass on a fresh database #9224

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions backend/plugins/bitbucket/e2e/comment_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ func TestCommentDataFlow(t *testing.T) {
)

// verify comment conversion
// the issue comment convertor joins _tool_bitbucket_issues
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_bitbucket_issues.csv", &models.BitbucketIssue{})
dataflowTester.FlushTabler(&ticket.IssueComment{})
dataflowTester.FlushTabler(&code.PullRequestComment{})
dataflowTester.Subtask(tasks.ConvertIssueCommentsMeta, taskData)
Expand Down
1 change: 1 addition & 0 deletions backend/plugins/customize/e2e/import_issue_changelogs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ func TestImportIssueChangelogDataFlow(t *testing.T) {
// 清空表
dataflowTester.FlushTabler(&ticket.IssueChangelogs{})
dataflowTester.FlushTabler(&crossdomain.Account{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})

// 初始化服务
svc := service.NewService(dataflowTester.Dal)
Expand Down
2 changes: 2 additions & 0 deletions backend/plugins/customize/e2e/import_issue_commits_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"testing"

"github.com/apache/devlake/core/models/domainlayer/crossdomain"
"github.com/apache/devlake/core/models/domainlayer/ticket"
"github.com/apache/devlake/helpers/e2ehelper"
"github.com/apache/devlake/plugins/customize/impl"
"github.com/apache/devlake/plugins/customize/service"
Expand All @@ -33,6 +34,7 @@ func TestImportIssueCommitDataFlow(t *testing.T) {

// create table `issue_commits`
dataflowTester.FlushTabler(&crossdomain.IssueCommit{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})
svc := service.NewService(dataflowTester.Dal)

f, err1 := os.Open("raw_tables/issues_commits.csv")
Expand Down
1 change: 1 addition & 0 deletions backend/plugins/customize/e2e/import_issue_worklogs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ func TestImportIssueWorklogDataFlow(t *testing.T) {
// 清空表
dataflowTester.FlushTabler(&ticket.IssueWorklog{})
dataflowTester.FlushTabler(&crossdomain.Account{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})

// 初始化服务
svc := service.NewService(dataflowTester.Dal)
Expand Down
2 changes: 2 additions & 0 deletions backend/plugins/dora/e2e/calculate_change_lead_time_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ func TestCalculateCLTimeDataFlow(t *testing.T) {
dataflowTester.ImportNullableCsvIntoTabler("./change_lead_time/commits_diffs.csv", &code.CommitsDiff{})
dataflowTester.ImportCsvIntoTabler("./change_lead_time/pull_request_comments.csv", &code.PullRequestComment{})
dataflowTester.ImportCsvIntoTabler("./change_lead_time/pull_request_commits.csv", &code.PullRequestCommit{})
// bot1 comments on pr1 before the first human review; it must not be counted as the first review
dataflowTester.ImportCsvIntoTabler("./change_lead_time/accounts.csv", &crossdomain.Account{})

// verify converter
dataflowTester.FlushTabler(&crossdomain.ProjectPrMetric{})
Expand Down
3 changes: 3 additions & 0 deletions backend/plugins/dora/e2e/change_lead_time/accounts.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
id,user_name,is_bot
b,reviewer-b,0
bot1,ci-bot,1
1 change: 1 addition & 0 deletions backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
id,pull_request_id,created_date,account_id
comment00,pr0,2022-1-11 8:51:47,a
comment01_bot,pr1,2023-4-11 4:52:47,bot1
comment01,pr1,2023-4-11 4:53:47,a
comment02,pr1,2023-4-11 4:56:47,b
comment03,pr1,2023-4-11 4:59:47,c
Expand Down
3 changes: 3 additions & 0 deletions backend/plugins/github/e2e/milestone_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ func TestMilestoneDataFlow(t *testing.T) {

dataflowTester.FlushTabler(&models.GithubMilestone{})
dataflowTester.FlushTabler(&models.GithubIssue{})
dataflowTester.FlushTabler(&models.GithubIssueLabel{})
dataflowTester.FlushTabler(&models.GithubRepoAccount{})
dataflowTester.FlushTabler(&models.GithubIssueAssignee{})

// import raw data table
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_github_api_milestones.csv", "_raw_"+tasks.RAW_MILESTONE_TABLE)
Expand Down
1 change: 1 addition & 0 deletions backend/plugins/github/e2e/pr_review_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func TestPrReviewDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.GithubReviewer{})
dataflowTester.FlushTabler(&models.GithubPrReview{})
dataflowTester.FlushTabler(&models.GithubRepoAccount{})
dataflowTester.Subtask(tasks.ExtractApiPullRequestReviewsMeta, taskData)
dataflowTester.VerifyTable(
models.GithubPrReview{},
Expand Down
2 changes: 2 additions & 0 deletions backend/plugins/gitlab/e2e/issues_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ func TestGitlabIssueDataFlow(t *testing.T) {
// verify conversion
dataflowTester.FlushTabler(&ticket.Issue{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})
dataflowTester.FlushTabler(&ticket.IssueLabel{})
dataflowTester.FlushTabler(&ticket.IssueAssignee{})
dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData)
dataflowTester.VerifyTable(
ticket.Issue{},
Expand Down
3 changes: 3 additions & 0 deletions backend/plugins/gitlab/e2e/mr_detail_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ func TestGitlabMrDetailDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.GitlabMergeRequest{})
dataflowTester.FlushTabler(&models.GitlabMrLabel{})
dataflowTester.FlushTabler(&models.GitlabReviewer{})
dataflowTester.FlushTabler(&models.GitlabAssignee{})
dataflowTester.Subtask(tasks.ExtractApiMergeRequestDetailsMeta, taskData)
dataflowTester.VerifyTableWithOptions(&models.GitlabMergeRequest{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/_tool_gitlab_merge_request_details.csv",
Expand All @@ -61,6 +63,7 @@ func TestGitlabMrDetailDataFlow(t *testing.T) {

// verify conversion
dataflowTester.FlushTabler(&code.PullRequest{})
dataflowTester.FlushTabler(&code.PullRequestLabel{})
dataflowTester.Subtask(tasks.ConvertApiMergeRequestsMeta, taskData)
dataflowTester.VerifyTableWithOptions(&code.PullRequest{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/pull_requests.csv",
Expand Down
2 changes: 2 additions & 0 deletions backend/plugins/gitlab/e2e/mr_notes_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ func TestGitlabMrNoteDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.GitlabMergeRequest{})
dataflowTester.FlushTabler(&models.GitlabMrLabel{})
dataflowTester.FlushTabler(&models.GitlabReviewer{})
dataflowTester.FlushTabler(&models.GitlabAssignee{})
dataflowTester.Subtask(tasks.ExtractApiMergeRequestsMeta, taskData)
dataflowTester.VerifyTable(
models.GitlabMergeRequest{},
Expand Down
5 changes: 5 additions & 0 deletions backend/plugins/gitlab/e2e/mr_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ func TestGitlabMrDataFlow(t *testing.T) {
// verify extraction
dataflowTester.FlushTabler(&models.GitlabMergeRequest{})
dataflowTester.FlushTabler(&models.GitlabMrLabel{})
dataflowTester.FlushTabler(&models.GitlabReviewer{})
dataflowTester.FlushTabler(&models.GitlabAssignee{})
dataflowTester.Subtask(tasks.ExtractApiMergeRequestsMeta, taskData)
dataflowTester.VerifyTableWithOptions(&models.GitlabMergeRequest{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/_tool_gitlab_merge_requests.csv",
Expand All @@ -59,6 +61,9 @@ func TestGitlabMrDataFlow(t *testing.T) {

// verify conversion
dataflowTester.FlushTabler(&code.PullRequest{})
dataflowTester.FlushTabler(&code.PullRequestAssignee{})
dataflowTester.FlushTabler(&code.PullRequestReviewer{})
dataflowTester.FlushTabler(&code.PullRequestLabel{})
dataflowTester.Subtask(tasks.ConvertApiMergeRequestsMeta, taskData)
dataflowTester.VerifyTableWithOptions(&code.PullRequest{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/pull_requests.csv",
Expand Down
2 changes: 2 additions & 0 deletions backend/plugins/jira/e2e/epic_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ func TestEpicDataflow(t *testing.T) {
dataflowTester.FlushTabler(&models.JiraAccount{})
dataflowTester.FlushTabler(&models.JiraIssueType{})
dataflowTester.FlushTabler(&models.JiraIssueComment{})
dataflowTester.FlushTabler(&models.JiraIssueLabel{})
dataflowTester.FlushTabler(&models.JiraIssueField{})

ctx := dataflowTester.SubtaskContext(taskData)

Expand Down
1 change: 1 addition & 0 deletions backend/plugins/jira/e2e/issue_relationship_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ func TestIssueRelationshipDataFlow(t *testing.T) {
dataflowTester.FlushTabler(&models.JiraAccount{})
dataflowTester.FlushTabler(&models.JiraIssueType{})
dataflowTester.FlushTabler(&models.JiraIssueLabel{})
dataflowTester.FlushTabler(&models.JiraIssueField{})
dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData)

dataflowTester.VerifyTableWithOptions(&models.JiraIssueRelationship{}, e2ehelper.TableOptions{
Expand Down
13 changes: 13 additions & 0 deletions backend/plugins/jira/e2e/issue_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,10 +281,23 @@ func TestIssueSubtaskWithoutMapping(t *testing.T) {
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issues.csv", "_raw_jira_api_issues")
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issue_types.csv", "_raw_jira_api_issue_types")

dataflowTester.FlushTabler(&models.JiraIssue{})
dataflowTester.FlushTabler(&models.JiraBoardIssue{})
dataflowTester.FlushTabler(&models.JiraSprintIssue{})
dataflowTester.FlushTabler(&models.JiraIssueComment{})
dataflowTester.FlushTabler(&models.JiraIssueChangelogs{})
dataflowTester.FlushTabler(&models.JiraIssueChangelogItems{})
dataflowTester.FlushTabler(&models.JiraWorklog{})
dataflowTester.FlushTabler(&models.JiraAccount{})
dataflowTester.FlushTabler(&models.JiraIssueType{})
dataflowTester.FlushTabler(&models.JiraIssueLabel{})
dataflowTester.FlushTabler(&models.JiraIssueField{})
dataflowTester.Subtask(tasks.ExtractIssueTypesMeta, taskData)
dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData)

dataflowTester.FlushTabler(&ticket.Issue{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})
dataflowTester.FlushTabler(&ticket.IssueAssignee{})

dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData)
dataflowTester.VerifyTable(
Expand Down
2 changes: 2 additions & 0 deletions backend/plugins/linear/e2e/issue_history_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ func TestLinearIssueHistoryDataFlow(t *testing.T) {

// verify conversion: tool layer -> domain layer
dataflowTester.FlushTabler(&ticket.IssueChangelogs{})
// the convertor also updates issues.lead_time_minutes
dataflowTester.FlushTabler(&ticket.Issue{})
dataflowTester.Subtask(tasks.ConvertIssueHistoryMeta, taskData)
dataflowTester.VerifyTableWithOptions(ticket.IssueChangelogs{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/issue_changelogs.csv",
Expand Down
1 change: 1 addition & 0 deletions backend/plugins/opsgenie/e2e/incident_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ func TestIncidentDataFlow(t *testing.T) {
)
dataflowTester.FlushTabler(&ticket.Issue{})
dataflowTester.FlushTabler(&ticket.IssueAssignee{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})
dataflowTester.Subtask(tasks.ConvertIncidentsMeta, taskData)
dataflowTester.VerifyTableWithOptions(
ticket.Issue{},
Expand Down
1 change: 1 addition & 0 deletions backend/plugins/pagerduty/e2e/incident_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ func TestIncidentDataFlow(t *testing.T) {
)
dataflowTester.FlushTabler(&ticket.Issue{})
dataflowTester.FlushTabler(&ticket.IssueAssignee{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})
dataflowTester.Subtask(tasks.ConvertIncidentsMeta, taskData)
dataflowTester.VerifyTableWithOptions(
ticket.Issue{},
Expand Down
7 changes: 7 additions & 0 deletions backend/plugins/tapd/e2e/bugs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,11 @@ func TestTapdBugCustomizeDueDate(t *testing.T) {
},
}

dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_tapd_bug_statuses.csv", &models.TapdBugStatus{})
dataflowTester.FlushTabler(&models.TapdBug{})
dataflowTester.FlushTabler(&models.TapdWorkSpaceBug{})
dataflowTester.FlushTabler(&models.TapdIterationBug{})
dataflowTester.FlushTabler(&models.TapdBugLabel{})
// import raw data table
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_tapd_api_bugs_for_due_date.csv", "_raw_tapd_api_bugs")
dataflowTester.Subtask(tasks.ExtractBugMeta, taskData)
Expand All @@ -198,6 +202,9 @@ func TestTapdBugCustomizeDueDate(t *testing.T) {
})

dataflowTester.FlushTabler(&ticket.Issue{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})
dataflowTester.FlushTabler(&ticket.SprintIssue{})
dataflowTester.FlushTabler(&ticket.IssueAssignee{})
dataflowTester.Subtask(tasks.ConvertBugMeta, taskData)
dataflowTester.VerifyTable(
ticket.Issue{},
Expand Down
8 changes: 8 additions & 0 deletions backend/plugins/tapd/e2e/stories_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,17 +159,25 @@ func TestTapdStoryCustomizeDueDate(t *testing.T) {
},
},
}
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_tapd_workitem_types.csv", &models.TapdWorkitemType{})
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_tapd_story_statuses.csv", &models.TapdStoryStatus{})
// import raw data table
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_tapd_api_stories_for_due_date.csv",
"_raw_tapd_api_stories")
dataflowTester.FlushTabler(&models.TapdStory{})
dataflowTester.FlushTabler(&models.TapdWorkSpaceStory{})
dataflowTester.FlushTabler(&models.TapdIterationStory{})
dataflowTester.FlushTabler(&models.TapdStoryLabel{})
dataflowTester.Subtask(tasks.ExtractStoryMeta, taskData)
dataflowTester.VerifyTableWithOptions(&models.TapdStory{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/_tool_tapd_stories_for_due_date.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
})

dataflowTester.FlushTabler(&ticket.Issue{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})
dataflowTester.FlushTabler(&ticket.SprintIssue{})
dataflowTester.FlushTabler(&ticket.IssueAssignee{})
dataflowTester.Subtask(tasks.ConvertStoryMeta, taskData)
dataflowTester.VerifyTableWithOptions(&ticket.Issue{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/issues_story_for_due_date.csv",
Expand Down
6 changes: 6 additions & 0 deletions backend/plugins/tapd/e2e/tasks_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,13 +154,19 @@ func TestTapdTaskCustomizeDueDate(t *testing.T) {

// verify extraction
dataflowTester.FlushTabler(&models.TapdTask{})
dataflowTester.FlushTabler(&models.TapdWorkSpaceTask{})
dataflowTester.FlushTabler(&models.TapdIterationTask{})
dataflowTester.FlushTabler(&models.TapdTaskLabel{})
dataflowTester.Subtask(tasks.ExtractTaskMeta, taskData)
dataflowTester.VerifyTableWithOptions(&models.TapdTask{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/_tool_tapd_tasks_for_due_date.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
})

dataflowTester.FlushTabler(&ticket.Issue{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})
dataflowTester.FlushTabler(&ticket.SprintIssue{})
dataflowTester.FlushTabler(&ticket.IssueAssignee{})
dataflowTester.Subtask(tasks.ConvertTaskMeta, taskData)
dataflowTester.VerifyTableWithOptions(&ticket.Issue{}, e2ehelper.TableOptions{
CSVRelPath: "./snapshot_tables/issues_task_for_due_date.csv",
Expand Down
1 change: 1 addition & 0 deletions backend/plugins/teambition/e2e/sprint_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ func TestTeambitionSprints(t *testing.T) {
)

dataflowTester.FlushTabler(&ticket.Sprint{})
dataflowTester.FlushTabler(&ticket.BoardSprint{})
dataflowTester.Subtask(tasks.ConvertSprintsMeta, taskData)
dataflowTester.VerifyTableWithOptions(
ticket.Sprint{},
Expand Down
2 changes: 2 additions & 0 deletions backend/plugins/teambition/e2e/task_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@ func TestTeambitionTask(t *testing.T) {

dataflowTester.FlushTabler(&ticket.IssueAssignee{})
dataflowTester.FlushTabler(&ticket.Issue{})
dataflowTester.FlushTabler(&ticket.BoardIssue{})
dataflowTester.FlushTabler(&ticket.SprintIssue{})
dataflowTester.Subtask(tasks.ConvertTasksMeta, taskData)
dataflowTester.VerifyTableWithOptions(
ticket.Issue{},
Expand Down