(fix/refactor): Remove unnecessary calls to Notification APIs (IDETECT-4992) (#1708)

* Notification APIs: remove unnecessary calls

* Don't wait at BOM level for IAC or impact analysis scans since we cannot check their completion status via notifications or bom status. This is existing behaviour, setting wait to true was a noop.

* Refactor Impact Analysis upload operation to createCodeLocationsWithoutNotificationTaskRange. Prevents call to notifications API for which the results were never used downstream, CodeLocationCreationData's NotificationTaskRange is null.

* Latest IntelligentModeStepRunner changes

* Pkg mngr and signature updates

* Testing for regressions

* Remove unused impact analysis related methods

* Remove unused ImpactAnalysisToolResult class

* Deprecate shouldWaitAtScanLevel since it simply checks if a server version exists and is at least 2023.1.1 which has reached end of service.

* Update bd-common version with soon to be released lib version

* Propogate removal of shouldWaitAtScanLevel to signature scanner

* Remove prescass pkg mngr BDIO code location upload from being considered a waitable (via notifications).

* Minor updates

* Rename back to uploadBdioFiles

* Clean up comments

* Revert changes to waitable signature scanner code loctaion

* Remove comments in signaturescansteprunner

* Add missed method update for binary upload

* Remove NotificationTaskRange param from WaitableCodeLocationData constructor as it is no longer used.

* Fix whitespace in diff

* Fix method rename issue after rebase

* Fix regression wrt correlated scanning. Correlated scan count calculations are tightly coupled with the legacy idea of a waitable code location.

* Bump bd-common version 68.0.0

Author: shantyk

shared-version.properties

@@ -1,2 +1,2 @@
-gradle.ext.blackDuckCommonVersion='67.0.20'
+gradle.ext.blackDuckCommonVersion='68.0.0'
 gradle.ext.springBootVersion='2.7.12'

src/main/java/com/blackduck/integration/detect/lifecycle/boot/product/ProductBoot.java

@@ -117,7 +117,6 @@ private BlackDuckRunData getBlackDuckRunData(
 
             BlackDuckServicesFactory blackDuckServicesFactory = blackDuckConnectivityResult.getBlackDuckServicesFactory();
             setBlackDuckVersionLevel(blackDuckServicesFactory, blackDuckConnectivityResult);
-            boolean waitAtScanLevel = shouldWaitAtScanLevel(blackDuckConnectivityResult);
 
             // Fetch server detect properties settings
             Optional<DetectPropertiesSetting> serverDetectProperties = fetchDetectPropertiesFromServer(
@@ -126,7 +125,7 @@ private BlackDuckRunData getBlackDuckRunData(
                 blackDuckServicesFactory.getBlackDuckApiClient()
             );
 
-            return createBlackDuckRunDataBasedOnPhoneHomeDecision(blackDuckDecision, blackDuckServicesFactory, blackDuckConnectivityResult, waitAtScanLevel, serverDetectProperties);
+            return createBlackDuckRunDataBasedOnPhoneHomeDecision(blackDuckDecision, blackDuckServicesFactory, blackDuckConnectivityResult, serverDetectProperties);
         } else {
             if (productBootOptions.isIgnoreConnectionFailures()) {
                 logger.info(String.format("Failed to connect to Black Duck SCA: %s", blackDuckConnectivityResult.getFailureReason()));
@@ -148,7 +147,6 @@ private BlackDuckRunData createBlackDuckRunDataBasedOnPhoneHomeDecision(
         BlackDuckDecision blackDuckDecision,
         BlackDuckServicesFactory blackDuckServicesFactory,
         BlackDuckConnectivityResult blackDuckConnectivityResult,
-        boolean waitAtScanLevel,
         Optional<DetectPropertiesSetting> serverDetectProperties
     ) {
         if (shouldUsePhoneHome(analyticsConfigurationService, blackDuckServicesFactory.getApiDiscovery(), blackDuckServicesFactory.getBlackDuckApiClient())) {
@@ -160,7 +158,6 @@ private BlackDuckRunData createBlackDuckRunDataBasedOnPhoneHomeDecision(
                     blackDuckServicesFactory,
                     phoneHomeManager,
                     blackDuckConnectivityResult,
-                    waitAtScanLevel,
                     serverDetectProperties
                 );
             } catch (IntegrationException e) {
@@ -171,7 +168,7 @@ private BlackDuckRunData createBlackDuckRunDataBasedOnPhoneHomeDecision(
         } else {
             logger.debug("Skipping phone home due to Black Duck SCA global settings.");
         }
-        return BlackDuckRunData.onlineNoPhoneHome(blackDuckDecision.scanMode(), blackDuckServicesFactory, blackDuckConnectivityResult, waitAtScanLevel, serverDetectProperties);
+        return BlackDuckRunData.onlineNoPhoneHome(blackDuckDecision.scanMode(), blackDuckServicesFactory, blackDuckConnectivityResult, serverDetectProperties);
     }
 
     private Optional<DetectPropertiesSetting> fetchDetectPropertiesFromServer(
@@ -210,18 +207,4 @@ private boolean shouldUsePhoneHome(AnalyticsConfigurationService analyticsConfig
             return true; // Skip phone home will be applied at the library level.
         }
     }
-    
-    private boolean shouldWaitAtScanLevel(BlackDuckConnectivityResult blackDuckConnectivityResult) {
-        BlackDuckVersionParser parser = new BlackDuckVersionParser();
-        Optional<BlackDuckVersion> blackDuckServerVersion = parser.parse(blackDuckConnectivityResult.getContactedServerVersion());
-        BlackDuckVersion minVersion = new BlackDuckVersion(2023, 1, 1);
-        
-        boolean waitAtScanLevel = false;
-        
-        if (blackDuckServerVersion.isPresent() && blackDuckServerVersion.get().isAtLeast(minVersion)) {
-            waitAtScanLevel = true;
-        }
-        
-        return waitAtScanLevel;
-    }
 }

src/main/java/com/blackduck/integration/detect/lifecycle/run/data/BlackDuckRunData.java

@@ -16,7 +16,6 @@ public class BlackDuckRunData {
     private final BlackDuckServerConfig blackDuckServerConfig;
     private final BlackDuckServicesFactory blackDuckServicesFactory;
     private final BlackduckScanMode scanMode;
-    private final boolean waitAtScanLevel;
     private final Optional<DetectPropertiesSetting> serverDetectProperties;
     private Optional<BlackDuckVersion> blackDuckServerVersion;
 
@@ -25,7 +24,6 @@ private BlackDuckRunData() {
         this.blackDuckServerConfig = null;
         this.blackDuckServicesFactory = null;
         this.scanMode = null;
-        this.waitAtScanLevel = false;
         this.serverDetectProperties = Optional.empty();
         this.blackDuckServerVersion = Optional.empty();
     }
@@ -35,16 +33,14 @@ protected BlackDuckRunData(
         BlackDuckConnectivityResult blackDuckConnectivityResult,
         BlackDuckServicesFactory blackDuckServicesFactory,
         BlackduckScanMode scanMode,
-        boolean waitAtScanLevel,
         Optional<DetectPropertiesSetting> serverDetectProperties
     ) {
         this.phoneHomeManager = phoneHomeManager;
         this.blackDuckServerConfig = blackDuckConnectivityResult != null ? blackDuckConnectivityResult.getBlackDuckServerConfig() : null;
         this.blackDuckServicesFactory = blackDuckServicesFactory;
         this.scanMode = scanMode;
-        this.waitAtScanLevel = waitAtScanLevel;
-        this.serverDetectProperties = serverDetectProperties;
 
+        this.serverDetectProperties = serverDetectProperties;
         determineBlackDuckServerVersion(blackDuckConnectivityResult);
     }
 
@@ -73,20 +69,18 @@ public static BlackDuckRunData online(
         BlackDuckServicesFactory blackDuckServicesFactory,
         PhoneHomeManager phoneHomeManager,
         BlackDuckConnectivityResult blackDuckConnectivityResult,
-        boolean waitAtScanLevel,
         Optional<DetectPropertiesSetting> serverDetectProperties
     ) {
-        return new BlackDuckRunData(phoneHomeManager, blackDuckConnectivityResult, blackDuckServicesFactory, scanMode, waitAtScanLevel, serverDetectProperties);
+        return new BlackDuckRunData(phoneHomeManager, blackDuckConnectivityResult, blackDuckServicesFactory, scanMode, serverDetectProperties);
     }
 
     public static BlackDuckRunData onlineNoPhoneHome(
         BlackduckScanMode scanMode,
         BlackDuckServicesFactory blackDuckServicesFactory,
         BlackDuckConnectivityResult blackDuckConnectivityResult,
-        boolean waitAtScanLevel,
         Optional<DetectPropertiesSetting> serverDetectProperties
     ) {
-        return new BlackDuckRunData(null, blackDuckConnectivityResult, blackDuckServicesFactory, scanMode, waitAtScanLevel, serverDetectProperties);
+        return new BlackDuckRunData(null, blackDuckConnectivityResult, blackDuckServicesFactory, scanMode, serverDetectProperties);
     }
 
     public Boolean isNonPersistent() {
@@ -96,10 +90,6 @@ public Boolean isNonPersistent() {
     public BlackduckScanMode getScanMode() {
         return scanMode;
     }
-    
-    public boolean shouldWaitAtScanLevel() {
-        return waitAtScanLevel;
-    }
 
     public Optional<BlackDuckVersion> getBlackDuckServerVersion() {
         return blackDuckServerVersion;

src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java

@@ -710,7 +710,7 @@ private File createProtobufHeaderFile(String type, NameVersion projectNameVersio
         }
     }
 
-    public void uploadBdioEntries(BlackDuckRunData blackDuckRunData, UUID bdScanId) throws IntegrationException, IOException {
+    public void uploadBdioEntriesForRapidMode(BlackDuckRunData blackDuckRunData, UUID bdScanId) throws IntegrationException, IOException {
         // parse directory and upload all chunks
         File bdioDirectory = directoryManager.getBdioOutputDirectory();
 
@@ -1011,7 +1011,7 @@ public void attemptToGenerateComponentLocationAnalysisIfEnabled() throws Operati
     //Post actions
     //End post actions
 
-    public final BdioUploadResult uploadBdioIntelligentPersistent(BlackDuckRunData blackDuckRunData, BdioResult bdioResult, Long timeout, String scassScanId) throws OperationException {
+    public final BdioUploadResult uploadBdioForIntelligentPersistentMode(BlackDuckRunData blackDuckRunData, BdioResult bdioResult, Long timeout, String scassScanId) throws OperationException {
         return auditLog.namedPublic(
             "Upload Intelligent Persistent Bdio",
             () -> new IntelligentPersistentUploadOperation(
@@ -1021,8 +1021,8 @@ public final BdioUploadResult uploadBdioIntelligentPersistent(BlackDuckRunData b
         );
     }
 
-    public final CodeLocationWaitData calculateCodeLocationWaitData(List<WaitableCodeLocationData> codeLocationCreationDatas) throws OperationException {
-        return auditLog.namedInternal("Calculate Code Location Wait Data", () -> new CodeLocationWaitCalculator().calculateWaitData(codeLocationCreationDatas));
+    public final CodeLocationWaitData calculateCodeLocationWaitDataGivenNotificationRangeStart(List<WaitableCodeLocationData> codeLocationCreationDatas, long codeLocationsUploadStartTime) throws OperationException {
+        return auditLog.namedInternal("Calculate Code Location Wait Data", () -> new CodeLocationWaitCalculator().calculateWaitData(codeLocationCreationDatas, codeLocationsUploadStartTime));
     }
 
     public final void publishCodeLocationData(Set<FormattedCodeLocation> codeLocationData) throws OperationException {
@@ -1297,13 +1297,6 @@ public ScanBatchRunner createScanBatchRunnerFromLocalInstall(File installDirecto
         });
     }
 
-    public NotificationTaskRange createCodeLocationRange(BlackDuckRunData blackDuckRunData) throws OperationException {
-        return auditLog.namedInternal(
-            "Create Code Location Task Range",
-            () -> blackDuckRunData.getBlackDuckServicesFactory().createCodeLocationCreationService().calculateCodeLocationRange()
-        );
-    }
-
     public SignatureScanOuputResult signatureScan(ScanBatch scanBatch, ScanBatchRunner scanBatchRunner) throws OperationException {
         return auditLog.namedPublic("Execute Signature Scan CLI", "SigScan", () -> new SignatureScanOperation().performScanActions(scanBatch, scanBatchRunner));
     }
@@ -1321,13 +1314,12 @@ public void publishSignatureScanReport(List<SignatureScannerReport> report) thro
     }
 
     public SignatureScannerCodeLocationResult calculateWaitableSignatureScannerCodeLocations(
-        NotificationTaskRange notificationTaskRange,
         List<SignatureScannerReport> reports
     ) throws OperationException {
         return auditLog.namedInternal(
             "Calculate Signature Scanner Waitable Code Locations",
             () -> new CalculateWaitableSignatureScanCodeLocations()
-                .calculateWaitableCodeLocations(notificationTaskRange, reports)
+                .calculateWaitableCodeLocations(reports)
         );
     }
 

src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IacScanStepRunner.java

@@ -96,7 +96,7 @@ private IacScanReport performOnlineScan(
         try {
             File resultsFile = operationRunner.performIacScanScan(scanTarget, iacScanExe, count);
             String codeLocationName = operationRunner.createIacScanCodeLocationName(scanTarget, projectNameVersion);
-            String scanId = initiateScan(detectRunUuid, projectNameVersion, blackDuckRunData.getBlackDuckServicesFactory().createBdio2FileUploadService(), codeLocationName
+            String scanId = initiateIacScan(detectRunUuid, projectNameVersion, blackDuckRunData.getBlackDuckServicesFactory().createBdio2FileUploadService(), codeLocationName
             );
             operationRunner.uploadIacScanResults(blackDuckRunData, resultsFile, scanId);
             return IacScanReport.SUCCESS_ONLINE(scanTarget, codeLocationName);
@@ -121,7 +121,7 @@ public IacScanReport performOfflineScan(File scanTarget, File iacScanExe, int co
     //TODO- look into extracting scan initiation to another class
 
     //TODO- this should only be necessary if we didn't already upload BDIO during DETECTORS phase
-    private String initiateScan(
+    private String initiateIacScan(
         String detectRunUuid,
         NameVersion projectNameVersion, Bdio2FileUploadService bdio2FileUploadService, String codeLocationNameOverride
     )