diff --git a/src/org/labkey/targetedms/TargetedMSController.java b/src/org/labkey/targetedms/TargetedMSController.java index 3a502baa5..09263b375 100644 --- a/src/org/labkey/targetedms/TargetedMSController.java +++ b/src/org/labkey/targetedms/TargetedMSController.java @@ -67,7 +67,6 @@ import org.labkey.api.action.SimpleErrorView; import org.labkey.api.action.SimpleViewAction; import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AdminUrls; import org.labkey.api.analytics.AnalyticsService; import org.labkey.api.attachments.DocumentConversionService; import org.labkey.api.attachments.SvgSource; @@ -106,7 +105,6 @@ import org.labkey.api.module.ModuleProperty; import org.labkey.api.pipeline.LocalDirectory; import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.pipeline.PipelineUrls; import org.labkey.api.pipeline.PipelineValidationException; @@ -141,7 +139,6 @@ import org.labkey.api.security.SecurityManager; import org.labkey.api.security.User; import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.ApplicationAdminPermission; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; @@ -151,7 +148,6 @@ import org.labkey.api.targetedms.TargetedMSUrls; import org.labkey.api.targetedms.model.QCMetricConfiguration; import org.labkey.api.targetedms.model.SampleFileInfo; -import org.labkey.api.util.ButtonBuilder; import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.ContainerContext; import org.labkey.api.util.DOM; @@ -229,7 +225,6 @@ import org.labkey.targetedms.parser.list.ListDefinition; import org.labkey.targetedms.parser.skyaudit.AuditLogEntry; import org.labkey.targetedms.parser.speclib.SpeclibReaderException; -import org.labkey.targetedms.pipeline.ChromatogramCrawlerJob; import org.labkey.targetedms.query.ChromatogramDisplayColumnFactory; import org.labkey.targetedms.query.ConflictResultsManager; import org.labkey.targetedms.query.GroupChromatogramsTableInfo; @@ -327,7 +322,6 @@ import static org.labkey.api.util.DOM.Attribute.height; import static org.labkey.api.util.DOM.Attribute.href; import static org.labkey.api.util.DOM.Attribute.id; -import static org.labkey.api.util.DOM.Attribute.method; import static org.labkey.api.util.DOM.Attribute.src; import static org.labkey.api.util.DOM.Attribute.style; import static org.labkey.api.util.DOM.Attribute.width; @@ -548,47 +542,6 @@ public static void addDashboardTab(String tab, Container c, String... includeWeb } } - public static class ChromatogramCrawlerForm - { - } - - @RequiresPermission(ApplicationAdminPermission.class) - public class ChromatogramCrawlerAction extends FormViewAction - { - @Override - public void validateCommand(ChromatogramCrawlerForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(ChromatogramCrawlerForm form, boolean reshow, BindException errors) - { - return new HtmlView("Chromatogram Crawler", DIV("Crawl all containers under the parent " + getContainer().getPath(), - DOM.LK.FORM(at(method, "POST"), - new ButtonBuilder("Start Crawl").submit(true).build()))); - } - - @Override - public boolean handlePost(ChromatogramCrawlerForm form, BindException errors) throws Exception - { - PipelineJob job = new ChromatogramCrawlerJob(getViewBackgroundInfo(), PipelineService.get().getPipelineRootSetting(ContainerManager.getRoot())); - PipelineService.get().queueJob(job); - return true; - } - - @Override - public URLHelper getSuccessURL(ChromatogramCrawlerForm form) - { - return urlProvider(PipelineUrls.class).urlBegin(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - urlProvider(AdminUrls.class).addAdminNavTrail(root, "Chromatogram Crawler", getClass(), getContainer()); - } - } - // ------------------------------------------------------------------------ // Action to create a Raw Data tab // ------------------------------------------------------------------------ diff --git a/src/org/labkey/targetedms/TargetedMSModule.java b/src/org/labkey/targetedms/TargetedMSModule.java index d4739e31e..8443a5df7 100644 --- a/src/org/labkey/targetedms/TargetedMSModule.java +++ b/src/org/labkey/targetedms/TargetedMSModule.java @@ -46,14 +46,11 @@ import org.labkey.api.protein.search.ProteinSearchForm; import org.labkey.api.query.QueryView; import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.ApplicationAdminPermission; -import org.labkey.api.settings.AdminConsole; import org.labkey.api.settings.OptionalFeatureFlag; import org.labkey.api.settings.OptionalFeatureService; import org.labkey.api.targetedms.TargetedMSService; import org.labkey.api.usageMetrics.UsageMetricsService; import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.view.ActionURL; import org.labkey.api.view.BaseWebPartFactory; import org.labkey.api.view.JspView; import org.labkey.api.view.NavTree; @@ -380,12 +377,12 @@ public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Po String inclSubFolders = (String)portalCtx.get("includeSubfolders"); if(!StringUtils.isBlank(inclSubFolders)) { - form.setIncludeSubfolders(Boolean.valueOf(inclSubFolders)); + form.setIncludeSubfolders(Boolean.parseBoolean(inclSubFolders)); } String hideIncludeSubfolder = (String)portalCtx.get("hideIncludeSubfolder"); if(!StringUtils.isBlank(hideIncludeSubfolder)) { - form.setHideIncludeSubfolders(Boolean.valueOf(hideIncludeSubfolder)); + form.setHideIncludeSubfolders(Boolean.parseBoolean(hideIncludeSubfolder)); } return new ModificationSearchWebPart(form); } @@ -651,9 +648,6 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) TargetedMSListener listener = new TargetedMSListener(); ContainerManager.addContainerListener(listener); - ActionURL chromatogramURL = new ActionURL(TargetedMSController.ChromatogramCrawlerAction.class, ContainerManager.getRoot()); - AdminConsole.addLink(AdminConsole.SettingsLinkType.Premium, "Targeted MS Chromatogram Crawler", chromatogramURL, ApplicationAdminPermission.class); - FileContentService fcs = FileContentService.get(); if(null != fcs) { diff --git a/src/org/labkey/targetedms/pipeline/ChromatogramCrawlerJob.java b/src/org/labkey/targetedms/pipeline/ChromatogramCrawlerJob.java deleted file mode 100644 index 5c32636a0..000000000 --- a/src/org/labkey/targetedms/pipeline/ChromatogramCrawlerJob.java +++ /dev/null @@ -1,335 +0,0 @@ -package org.labkey.targetedms.pipeline; - -import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.TableSelector; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.query.FieldKey; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.URLHelper; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.targetedms.TargetedMSManager; -import org.labkey.targetedms.TargetedMSRun; -import org.labkey.targetedms.TargetedMSSchema; -import org.labkey.targetedms.parser.Chromatogram; -import org.labkey.targetedms.parser.PrecursorChromInfo; -import org.labkey.targetedms.parser.SampleFile; -import org.labkey.targetedms.query.PrecursorManager; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -public class ChromatogramCrawlerJob extends PipelineJob -{ - private final CrawlType _crawlType = CrawlType.verifyOnly; - - /** For JSON serialization/deserialzation round-tripping - * @noinspection unused*/ - protected ChromatogramCrawlerJob() - { - - } - - public ChromatogramCrawlerJob(ViewBackgroundInfo info, @NotNull PipeRoot root) - { - super(TargetedMSPipelineProvider.name, info, root); - setLogFile(root.getRootFileLike().resolveChild(FileUtil.makeFileNameWithTimestamp("ChromatogramCrawl", "log"))); - - getLogger().info("Description of key statuses:"); - for (Chromatogram.SourceStatus value : Chromatogram.SourceStatus.values()) - { - getLogger().info("\t{}: {}", value.toString(), value.getDescription()); - } - getLogger().info("\n"); - } - - public enum CrawlType - { - verifyOnly - { - @Override - public SimpleFilter getFilter() - { - return new SimpleFilter(); - } - }, - /** Not supported yet, but coming soon */ - populateIndices - { - @Override - public SimpleFilter getFilter() - { - return new SimpleFilter(); - } - }; - - public abstract SimpleFilter getFilter(); - } - - @Override - public void run() - { - setStatus(TaskStatus.running); - var containers = ContainerManager.getAllChildren(getContainer(), getUser()); - - getLogger().info("Starting to process chromatogram rows over {} containers", containers.size()); - - CrawlSummary crawlSummary = new CrawlSummary(); - - int containerCount = 0; - // Loop through all containers - for (Container container : containers) - { - containerCount++; - - // See how many rows we have to process in this container - var schema = new TargetedMSSchema(getUser(), container); - var table = schema.getTable(TargetedMSSchema.TABLE_PRECURSOR_CHROM_INFO, ContainerFilter.current(this)); - var selector = new TableSelector(table, PageFlowUtil.set("Id", "SampleFileID"), _crawlType.getFilter(), new Sort(FieldKey.fromParts("SampleFileId"))); - long containerPCICount = selector.getRowCount(); - - if (containerPCICount == 0) - { - getLogger().info("Skipping container {} - no chromatogram rows to process", container.getPath()); - } - else - { - getLogger().info("Starting processing of container {} ({} of {}) with {} chromatogram rows to process", container.getPath(), containerCount, containers.size(), containerPCICount); - - var handler = new PrecursorChromInfoHandler(container); - try - { - // Use an uncached result set so that we can stream however many rows we have in the DB without - // blowing out the heap - try (ResultSet rs = selector.getResultSet(false, false)) - { - while (rs.next()) - { - handler.exec(rs.getInt("Id"), rs.getInt("SampleFileId")); - } - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - - for (RunSummary summary : handler._summaries) - { - getLogger().info(summary); - crawlSummary.add(summary); - } - } - } - getLogger().info("All done!"); - getLogger().info(crawlSummary.toString()); - setStatus(TaskStatus.complete); - } - - @Override - public URLHelper getStatusHref() - { - return null; - } - - @Override - public String getDescription() - { - return "Crawling chromatograms: " + _crawlType; - } - - private static class CrawlSummary - { - private int _dbOnlyCount; - private int _diskOnlyCount; - private int _noSkydResolvedCount; - private int _skydNotPresentCount; - private int _mismatchCount; - private int _matchCount; - private int _noChromatogramCount; - - public void add(RunSummary runSummary) - { - if (runSummary._dbOnlyCount > 0) - { - _dbOnlyCount++; - } - if (runSummary._diskOnlyCount > 0) - { - _diskOnlyCount++; - } - if (runSummary._noSkydResolvedCount > 0) - { - _noSkydResolvedCount++; - } - if (runSummary._skydNotPresentCount > 0) - { - _skydNotPresentCount++; - } - if (runSummary._mismatchCount > 0) - { - _mismatchCount++; - } - if (runSummary._matchCount > 0) - { - _matchCount++; - } - if (runSummary._noChromatogramCount > 0) - { - _noChromatogramCount++; - } - } - - @Override - public String toString() - { - return String.format("Crawl summary by Skyline document counts. Match: %3d, No SKYD resolved: %3d, DB only: %3d, Disk only: %3d, SKYD not present: %3d, Mismatch: %3d", - _matchCount, - _noSkydResolvedCount, - _dbOnlyCount, - _diskOnlyCount, - _skydNotPresentCount, - _mismatchCount); - } - } - - /** */ - private static class RunSummary - { - private final TargetedMSRun _run; - - private int _dbOnlyCount; - private int _diskOnlyCount; - private int _noSkydResolvedCount; - private int _skydNotPresentCount; - private int _mismatchCount; - private int _matchCount; - private int _noChromatogramCount; - private int _skippedCount; - - public RunSummary(TargetedMSRun run) - { - _run = run; - } - - @Override - public String toString() - { - // Count the number of different states we found for chromatograms, excluding the "skipped" one - int states = (_matchCount > 0 ? 1 : 0) + - (_noSkydResolvedCount > 0 ? 1 : 0) + - (_dbOnlyCount > 0 ? 1 : 0) + - (_diskOnlyCount > 0 ? 1 : 0) + - (_skydNotPresentCount > 0 ? 1 : 0) + - (_mismatchCount > 0 ? 1 : 0) + - (_noChromatogramCount > 0 ? 1 : 0); - - return String.format("Skyline document %7d. Match: %3d, No SKYD resolved: %3d, DB only: %3d, Disk only: %3d, SKYD not present: %3d, Mismatch: %3d, No chromatogram: %7d, Skipped: %7d" + (states > 1 ? " - MIXED STATES!!!" : ""), - _run.getId(), - _matchCount, - _noSkydResolvedCount, - _dbOnlyCount, - _diskOnlyCount, - _skydNotPresentCount, - _mismatchCount, - _noChromatogramCount, - _skippedCount); - } - } - - private static class PrecursorChromInfoHandler - { - private final Container _container; - private SampleFile _sampleFile; - private RunSummary _currentSummary; - private TargetedMSRun _run; - - private int _rowsForCurrentRun; - - private final List _summaries = new ArrayList<>(); - - public PrecursorChromInfoHandler(Container container) - { - _container = container; - } - - public void exec(int id, long sampleFileId) - { - // We're sorted by sample file, so keep - if (_sampleFile == null || _sampleFile.getId() != sampleFileId) - { - _sampleFile = TargetedMSManager.getSampleFile(sampleFileId, _container); - if (_sampleFile == null) - { - throw new IllegalStateException("Could not find sample file " + sampleFileId); - } - var replicate = TargetedMSManager.getReplicate(_sampleFile.getReplicateId(), _container); - if (replicate == null) - { - throw new IllegalStateException("Could not find replicate " + _sampleFile.getReplicateId()); - } - if (_run == null || replicate.getRunId() != _run.getId()) - { - _run = TargetedMSManager.getRun(replicate.getRunId()); - if (_run == null) - { - throw new IllegalStateException("Could not find run " + replicate.getRunId()); - } - _rowsForCurrentRun = 0; - - _currentSummary = new RunSummary(_run); - _summaries.add(_currentSummary); - } - } - - if (_rowsForCurrentRun++ < 5) - { - PrecursorChromInfo pci = PrecursorManager.getPrecursorChromInfo(_container, id); - Chromatogram chromatogram = pci.createChromatogram(_run); - if (chromatogram == null) - { - _currentSummary._noChromatogramCount++; - } - else - { - switch (chromatogram.getStatus()) - { - case dbOnly: - _currentSummary._dbOnlyCount++; - break; - case diskOnly: - _currentSummary._diskOnlyCount++; - break; - case noSkydResolved: - _currentSummary._noSkydResolvedCount++; - break; - case skydMissing: - _currentSummary._skydNotPresentCount++; - break; - case mismatch: - _currentSummary._mismatchCount++; - break; - case match: - _currentSummary._matchCount++; - break; - default: - throw new IllegalArgumentException("Unhandled status" + chromatogram.getStatus()); - } - } - } - else - { - _currentSummary._skippedCount++; - } - } - } -}