Skip to content

[PLUGIN-1931] Add necessary condition to prevent query job when appropriate parameters are empty #1584

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adrikagupta merged 1 commit into from
Oct 1, 2025
+107 −58
Merged

[PLUGIN-1931] Add necessary condition to prevent query job when appropriate parameters are empty #1584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/main/java/io/cdap/plugin/gcp/bigquery/source/PartitionedBigQueryInputFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,12 @@ String generateQuery(String partitionFromDate, String partitionToDate, String fi
String datasetProject, String dataset, String table, String limit, String orderBy,
Boolean isPartitionFilterRequired, StandardTableDefinition tableDefinition) {

if (Strings.isNullOrEmpty(filter) && Strings.isNullOrEmpty(orderBy) && Strings.isNullOrEmpty(
limit)
&& Strings.isNullOrEmpty(partitionFromDate) && Strings.isNullOrEmpty(partitionToDate)) {
return null;
Comment thread
adrikagupta marked this conversation as resolved.
}

RangePartitioning rangePartitioning = tableDefinition.getRangePartitioning();
TimePartitioning timePartitioning = tableDefinition.getTimePartitioning();
StringBuilder condition = new StringBuilder();
Expand Down
159 changes: 101 additions & 58 deletions src/test/java/io/cdap/plugin/gcp/bigquery/source/PartitionedBigQueryInputFormatTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,15 @@

@RunWith(MockitoJUnitRunner.class)
public class PartitionedBigQueryInputFormatTest {

private static final String TEST_PROJECT = "test-project";
private static final String TEST_DATASET = "test-dataset";
private static final String TEST_TABLE = "test-table";
private static final String TEST_FILTER = "age > 10";
private static final String TEST_LIMIT = "100";
private static final String TEST_ORDER_BY = "name asc";
private static final String TEST_TABLE_SPEC = String.format("%s.%s.%s", TEST_PROJECT, TEST_DATASET, TEST_TABLE);
private static final String TEST_TABLE_SPEC = String.format("%s.%s.%s", TEST_PROJECT,
TEST_DATASET, TEST_TABLE);
private static final String TEST_FROM_DATE = "2025-01-01";
private static final String TEST_TO_DATE = "2025-01-02";
private static final String TEST_PARTITION_CONDITION =
Expand Down Expand Up @@ -124,17 +126,6 @@ public void testGenerateQuery_WithFilterOnly() {
Assert.assertEquals(expectedQuery, generatedQuery);
}

@Test
public void testGenerateQuery_NoOptions() {
String expectedQuery = String.format("select * from %s", TEST_TABLE_SPEC);

String generatedQuery = format.generateQuery(null, null,
null, TEST_PROJECT, TEST_DATASET, TEST_TABLE,
null, null,
false, mockTableDefinition);
Assert.assertEquals(expectedQuery, generatedQuery);
}

@Test
public void testGenerateQuery_AllOptions() {
String expectedQuery = String.format("select * from %s where %s order by %s limit %s",
Expand Down Expand Up @@ -163,38 +154,6 @@ public void testGenerateQuery_TimePartitionWithDates() {
Assert.assertEquals(expectedQuery, generatedQuery);
}


@Test
public void testGenerateQuery_TimePartitionRequiredNoDates() {
when(mockTableDefinition.getTimePartitioning()).thenReturn(mockTimePartitioning);
when(mockTimePartitioning.getField()).thenReturn(null);

String expectedQuery = String.format("select * from %s where %s",
TEST_TABLE_SPEC, TEST_DEFAULT_TIME_CONDITION);

String generatedQuery = format.generateQuery(null, null,
null, TEST_PROJECT, TEST_DATASET, TEST_TABLE,
null, null,
true, mockTableDefinition);
Assert.assertEquals(expectedQuery, generatedQuery);
}

@Test
public void testGenerateQuery_RangePartitionRequiredNoDates() {
when(mockTableDefinition.getTimePartitioning()).thenReturn(null);
when(mockTableDefinition.getRangePartitioning()).thenReturn(mockRangePartitioning);
when(mockRangePartitioning.getField()).thenReturn("range_col");

String expectedQuery = String.format("select * from %s where %s",
TEST_TABLE_SPEC, TEST_DEFAULT_RANGE_CONDITION);

String generatedQuery = format.generateQuery(null, null, null,
TEST_PROJECT, TEST_DATASET, TEST_TABLE,
null, null,
true, mockTableDefinition);
Assert.assertEquals(expectedQuery, generatedQuery);
}

@Test
public void testGenerateQuery_TimePartitionRequiredAndFilter() {
when(mockTableDefinition.getTimePartitioning()).thenReturn(mockTimePartitioning);
Expand Down Expand Up @@ -229,22 +188,56 @@ public void testGenerateQuery_TimeUnitPartitionWithDates() {
}

@Test
public void testGenerateQuery_TimeUnitPartitionRequiredNoDates() {
public void testGenerateQuery_TimePartitionFilterNotRequiredWithDates() {
when(mockTableDefinition.getTimePartitioning()).thenReturn(mockTimePartitioning);
when(mockTimePartitioning.getField()).thenReturn(TEST_TIME_UNIT_COL);
when(mockTimePartitioning.getField()).thenReturn(null);

String expectedQuery = String.format("select * from %s where %s",
TEST_TABLE_SPEC, TEST_DEFAULT_TIME_UNIT_CONDITION);
String expectedQuery = String.format("select * from %s where (%s)",
TEST_TABLE_SPEC,
TEST_PARTITION_CONDITION);

String generatedQuery = format.generateQuery(null, null, null,
String generatedQuery = format.generateQuery(TEST_FROM_DATE, TEST_TO_DATE, null,
TEST_PROJECT, TEST_DATASET, TEST_TABLE,
null, null,
true, mockTableDefinition);
false, mockTableDefinition);
Assert.assertEquals(expectedQuery, generatedQuery);
}

@Test
public void testGenerateQuery_TimePartitionFilterNotRequiredWithDates() {
public void testGenerateQuery_NoOptions_ShouldReturnNull() {
String generatedQuery = format.generateQuery(null, null,
null, TEST_PROJECT, TEST_DATASET, TEST_TABLE,
null, null,
false, mockTableDefinition);
Assert.assertNull("Query should be null if no filters or options are set.", generatedQuery);
}

@Test
public void testGenerateQuery_WithLimitOnly_ShouldAssertQuery() {
String expectedQuery = String.format("select * from %s limit %s", TEST_TABLE_SPEC,
TEST_LIMIT);

String generatedQuery = format.generateQuery(null, null,
null, TEST_PROJECT, TEST_DATASET, TEST_TABLE,
TEST_LIMIT, null,
false, mockTableDefinition);
Assert.assertEquals(expectedQuery, generatedQuery);
}

@Test
public void testGenerateQuery_WithOrderByOnly_ShouldAssertQuery() {
String expectedQuery = String.format("select * from %s order by %s", TEST_TABLE_SPEC,
TEST_ORDER_BY);

String generatedQuery = format.generateQuery(null, null,
null, TEST_PROJECT, TEST_DATASET, TEST_TABLE,
null, TEST_ORDER_BY,
false, mockTableDefinition);
Assert.assertEquals(expectedQuery, generatedQuery);
}

@Test
public void testGenerateQuery_TimePartitionNotRequired_WithDates_ShouldAssertQuery() {
when(mockTableDefinition.getTimePartitioning()).thenReturn(mockTimePartitioning);
when(mockTimePartitioning.getField()).thenReturn(null);

Expand All @@ -260,27 +253,77 @@ public void testGenerateQuery_TimePartitionFilterNotRequiredWithDates() {
}

@Test
public void testGenerateQuery_TimePartitionFilterNotRequiredNoDates() {
public void testGenerateQuery_TimePartitionRequired_WithFilterOnly_ShouldAssertQuery() {
when(mockTableDefinition.getTimePartitioning()).thenReturn(mockTimePartitioning);
when(mockTimePartitioning.getField()).thenReturn(null);

String expectedQuery = String.format("select * from %s", TEST_TABLE_SPEC);
String generatedQuery = format.generateQuery(null, null, null,
String expectedQuery = String.format("select * from %s where %s and (%s)",
TEST_TABLE_SPEC, TEST_DEFAULT_TIME_CONDITION, TEST_FILTER);

String generatedQuery = format.generateQuery(null, null,
TEST_FILTER, TEST_PROJECT, TEST_DATASET, TEST_TABLE,
null, null,
true, mockTableDefinition);
Assert.assertEquals(expectedQuery, generatedQuery);
}

@Test
public void testGenerateQuery_RangePartitionRequiredAndFilter() {
when(mockTableDefinition.getRangePartitioning()).thenReturn(mockRangePartitioning);
when(mockRangePartitioning.getField()).thenReturn("range_col");

String expectedQuery = String.format("select * from %s where %s and (%s)",
TEST_TABLE_SPEC, TEST_DEFAULT_RANGE_CONDITION, TEST_FILTER);

String generatedQuery = format.generateQuery(null, null, TEST_FILTER,
TEST_PROJECT, TEST_DATASET, TEST_TABLE,
null, null,
false, mockTableDefinition);
true, mockTableDefinition);
Assert.assertEquals(expectedQuery, generatedQuery);
}

@Test
public void testGenerateQuery_RangePartitionFilterNotRequired() {
public void testGenerateQuery_RangePartitionRequiredWithLimit() {
when(mockTableDefinition.getRangePartitioning()).thenReturn(mockRangePartitioning);
when(mockRangePartitioning.getField()).thenReturn("range_col");

String expectedQuery = String.format("select * from %s", TEST_TABLE_SPEC);
String expectedQuery = String.format("select * from %s where %s limit %s",
TEST_TABLE_SPEC, TEST_DEFAULT_RANGE_CONDITION, TEST_LIMIT);

String generatedQuery = format.generateQuery(null, null, null,
TEST_PROJECT, TEST_DATASET, TEST_TABLE,
TEST_LIMIT, null,
true, mockTableDefinition);
Assert.assertEquals(expectedQuery, generatedQuery);
}

@Test
public void testGenerateQuery_TimeUnitPartitionRequiredAndFilter() {
when(mockTableDefinition.getTimePartitioning()).thenReturn(mockTimePartitioning);
when(mockTimePartitioning.getField()).thenReturn(TEST_TIME_UNIT_COL);

String expectedQuery = String.format("select * from %s where %s and (%s)",
TEST_TABLE_SPEC, TEST_DEFAULT_TIME_UNIT_CONDITION, TEST_FILTER);

String generatedQuery = format.generateQuery(null, null, TEST_FILTER,
TEST_PROJECT, TEST_DATASET, TEST_TABLE,
null, null,
false, mockTableDefinition);
true, mockTableDefinition);
Assert.assertEquals(expectedQuery, generatedQuery);
}

@Test
public void testGenerateQuery_TimeUnitPartitionRequiredWithLimit() {
when(mockTableDefinition.getTimePartitioning()).thenReturn(mockTimePartitioning);
when(mockTimePartitioning.getField()).thenReturn(TEST_TIME_UNIT_COL);

String expectedQuery = String.format("select * from %s where %s limit %s",
TEST_TABLE_SPEC, TEST_DEFAULT_TIME_UNIT_CONDITION, TEST_LIMIT);

String generatedQuery = format.generateQuery(null, null, null,
TEST_PROJECT, TEST_DATASET, TEST_TABLE,
TEST_LIMIT, null,
true, mockTableDefinition);
Assert.assertEquals(expectedQuery, generatedQuery);
}
}
Loading