extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/HisatServlet.java

Code
Comments
Other
Rev Date Author Line
4594 26 Sep 17 nicklas 1 package net.sf.basedb.reggie.servlet;
4594 26 Sep 17 nicklas 2
4594 26 Sep 17 nicklas 3 import java.io.IOException;
4594 26 Sep 17 nicklas 4 import java.util.ArrayList;
4594 26 Sep 17 nicklas 5 import java.util.List;
4594 26 Sep 17 nicklas 6
4594 26 Sep 17 nicklas 7 import javax.servlet.ServletException;
4594 26 Sep 17 nicklas 8 import javax.servlet.http.HttpServlet;
4594 26 Sep 17 nicklas 9 import javax.servlet.http.HttpServletRequest;
4594 26 Sep 17 nicklas 10 import javax.servlet.http.HttpServletResponse;
4594 26 Sep 17 nicklas 11
4594 26 Sep 17 nicklas 12 import org.json.simple.JSONArray;
4594 26 Sep 17 nicklas 13 import org.json.simple.JSONObject;
4594 26 Sep 17 nicklas 14
4594 26 Sep 17 nicklas 15 import net.sf.basedb.core.BasicItem;
4594 26 Sep 17 nicklas 16 import net.sf.basedb.core.ItemList;
4594 26 Sep 17 nicklas 17 import net.sf.basedb.core.DbControl;
4594 26 Sep 17 nicklas 18 import net.sf.basedb.core.DerivedBioAssay;
4594 26 Sep 17 nicklas 19 import net.sf.basedb.core.Extract;
4620 16 Nov 17 nicklas 20 import net.sf.basedb.core.Item;
4594 26 Sep 17 nicklas 21 import net.sf.basedb.core.ItemNotFoundException;
4594 26 Sep 17 nicklas 22 import net.sf.basedb.core.ItemQuery;
4594 26 Sep 17 nicklas 23 import net.sf.basedb.core.Job;
4594 26 Sep 17 nicklas 24 import net.sf.basedb.core.Protocol;
4594 26 Sep 17 nicklas 25 import net.sf.basedb.core.SessionControl;
4594 26 Sep 17 nicklas 26 import net.sf.basedb.core.Software;
4594 26 Sep 17 nicklas 27 import net.sf.basedb.core.Trashcan;
4594 26 Sep 17 nicklas 28 import net.sf.basedb.core.query.Annotations;
4594 26 Sep 17 nicklas 29 import net.sf.basedb.core.query.Expressions;
4594 26 Sep 17 nicklas 30 import net.sf.basedb.core.query.Hql;
4594 26 Sep 17 nicklas 31 import net.sf.basedb.core.query.Orders;
4594 26 Sep 17 nicklas 32 import net.sf.basedb.core.query.Restrictions;
4594 26 Sep 17 nicklas 33 import net.sf.basedb.core.snapshot.SnapshotManager;
4599 28 Sep 17 nicklas 34 import net.sf.basedb.opengrid.JobDefinition;
4594 26 Sep 17 nicklas 35 import net.sf.basedb.opengrid.OpenGridCluster;
4594 26 Sep 17 nicklas 36 import net.sf.basedb.opengrid.service.OpenGridService;
4594 26 Sep 17 nicklas 37 import net.sf.basedb.reggie.JsonUtil;
4594 26 Sep 17 nicklas 38 import net.sf.basedb.reggie.Reggie;
5384 26 Apr 19 nicklas 39 import net.sf.basedb.reggie.activity.ActivityDef;
4600 28 Sep 17 nicklas 40 import net.sf.basedb.reggie.autoconfirm.HisatAlignAutoConfirmer;
4594 26 Sep 17 nicklas 41 import net.sf.basedb.reggie.counter.CounterService;
4594 26 Sep 17 nicklas 42 import net.sf.basedb.reggie.dao.AlignedSequences;
4594 26 Sep 17 nicklas 43 import net.sf.basedb.reggie.dao.Annotationtype;
4594 26 Sep 17 nicklas 44 import net.sf.basedb.reggie.dao.BiomaterialList;
4594 26 Sep 17 nicklas 45 import net.sf.basedb.reggie.dao.Library;
4656 24 Jan 18 nicklas 46 import net.sf.basedb.reggie.dao.MaskedSequences;
4594 26 Sep 17 nicklas 47 import net.sf.basedb.reggie.dao.MergedSequences;
5543 06 Aug 19 nicklas 48 import net.sf.basedb.reggie.dao.Pipeline;
4594 26 Sep 17 nicklas 49 import net.sf.basedb.reggie.dao.ReggieRole;
4594 26 Sep 17 nicklas 50 import net.sf.basedb.reggie.dao.Rna;
4594 26 Sep 17 nicklas 51 import net.sf.basedb.reggie.dao.Subtype;
4594 26 Sep 17 nicklas 52 import net.sf.basedb.reggie.grid.HisatAlignJobCreator;
4599 28 Sep 17 nicklas 53 import net.sf.basedb.reggie.grid.ScriptUtil;
4594 26 Sep 17 nicklas 54 import net.sf.basedb.util.Values;
4594 26 Sep 17 nicklas 55 import net.sf.basedb.util.error.ThrowableUtil;
4594 26 Sep 17 nicklas 56
4594 26 Sep 17 nicklas 57
4594 26 Sep 17 nicklas 58 public class HisatServlet 
4594 26 Sep 17 nicklas 59   extends HttpServlet 
4594 26 Sep 17 nicklas 60 {
4594 26 Sep 17 nicklas 61
4594 26 Sep 17 nicklas 62   private static final long serialVersionUID = -8731078921144280328L;
4594 26 Sep 17 nicklas 63   
4594 26 Sep 17 nicklas 64   
4594 26 Sep 17 nicklas 65   public HisatServlet()
4594 26 Sep 17 nicklas 66   {}
4594 26 Sep 17 nicklas 67
4594 26 Sep 17 nicklas 68   @SuppressWarnings("unchecked")
4594 26 Sep 17 nicklas 69   @Override
4594 26 Sep 17 nicklas 70   protected void doGet(HttpServletRequest req, HttpServletResponse resp)
4594 26 Sep 17 nicklas 71     throws ServletException, IOException 
4594 26 Sep 17 nicklas 72   {
4594 26 Sep 17 nicklas 73     String cmd = req.getParameter("cmd");
4594 26 Sep 17 nicklas 74     JsonUtil.setJsonResponseHeaders(resp);
4594 26 Sep 17 nicklas 75     
4594 26 Sep 17 nicklas 76     JSONObject json = new JSONObject();
4594 26 Sep 17 nicklas 77     json.put("status", "ok");
4594 26 Sep 17 nicklas 78     
4594 26 Sep 17 nicklas 79     final SessionControl sc = Reggie.getSessionControl(req);
4594 26 Sep 17 nicklas 80     DbControl dc = null;
4594 26 Sep 17 nicklas 81     try
4594 26 Sep 17 nicklas 82     {
4594 26 Sep 17 nicklas 83       if ("GetMergedSequencesForHisat".equals(cmd))
4594 26 Sep 17 nicklas 84       {
6335 15 Jun 21 nicklas 85         dc = sc.newDbControl(":Start Hisat alignment");
4596 27 Sep 17 nicklas 86         
4989 28 Sep 18 nicklas 87         List<MergedSequences> list = null;
4989 28 Sep 18 nicklas 88         String items = Values.getStringOrNull(req.getParameter("items"));
4989 28 Sep 18 nicklas 89         if (items == null)
4989 28 Sep 18 nicklas 90         {
4989 28 Sep 18 nicklas 91           // Load items from the pipeline list
4989 28 Sep 18 nicklas 92           ItemList hisatPipeline = BiomaterialList.HISAT_PIPELINE.load(dc);
6183 26 Mar 21 nicklas 93           ItemQuery<DerivedBioAssay> query = hisatPipeline.getMembers();
4989 28 Sep 18 nicklas 94           query.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
4989 28 Sep 18 nicklas 95           Subtype.MERGED_SEQUENCES.addFilter(dc, query);
4989 28 Sep 18 nicklas 96           // Join LibPlate for sorting
4989 28 Sep 18 nicklas 97           query.join(Hql.innerJoin(null, "extract", "lib", true));
4989 28 Sep 18 nicklas 98           query.join(Hql.innerJoin("lib", "bioWell", "bw", true));
4989 28 Sep 18 nicklas 99           query.join(Hql.innerJoin("bw", "bioPlate", "libPlate", true));
4989 28 Sep 18 nicklas 100           query.order(Orders.asc(Hql.property("libPlate", "id")));
4989 28 Sep 18 nicklas 101           query.order(Orders.asc(Hql.property("name")));
4989 28 Sep 18 nicklas 102           query.setMaxResults(250);
4989 28 Sep 18 nicklas 103           list = MergedSequences.toList(query.list(dc));
4989 28 Sep 18 nicklas 104         }
4989 28 Sep 18 nicklas 105         else
4989 28 Sep 18 nicklas 106         {
4989 28 Sep 18 nicklas 107           Integer[] ids = Values.getInt(items.split(","));
4989 28 Sep 18 nicklas 108           list = new ArrayList<MergedSequences>();
4989 28 Sep 18 nicklas 109           for (Integer id : ids)
4989 28 Sep 18 nicklas 110           {
4989 28 Sep 18 nicklas 111             list.add(MergedSequences.getById(dc, id));
4989 28 Sep 18 nicklas 112           }
4989 28 Sep 18 nicklas 113         }
4594 26 Sep 17 nicklas 114
4594 26 Sep 17 nicklas 115         SnapshotManager manager = new SnapshotManager();
4594 26 Sep 17 nicklas 116         JSONArray jsonMergedSequences = new JSONArray();
4594 26 Sep 17 nicklas 117         for (MergedSequences ms : list)
4594 26 Sep 17 nicklas 118         {
5542 05 Aug 19 nicklas 119           ms.loadAnnotations(dc, "pipeline", Annotationtype.PIPELINE, null);
4594 26 Sep 17 nicklas 120           Library lib = ms.getLibrary(dc);
4594 26 Sep 17 nicklas 121           lib.loadBioPlateLocation();
4594 26 Sep 17 nicklas 122           Rna r = lib.getRna(dc, true);
5542 05 Aug 19 nicklas 123           if (r != null) lib.setAnnotation("specimen", r.loadYellowLabelInfo(dc, manager));
4594 26 Sep 17 nicklas 124           ms.setAnnotation("lib", lib.asJSONObject());
4594 26 Sep 17 nicklas 125           ms.loadAnnotations(dc, manager, "AutoProcess", Annotationtype.AUTO_PROCESSING, null);
4989 28 Sep 18 nicklas 126           ms.loadDoNotUseAnnotations(dc, manager);
4594 26 Sep 17 nicklas 127           jsonMergedSequences.add(ms.asJSONObject());
4594 26 Sep 17 nicklas 128         }
5547 07 Aug 19 nicklas 129         json.put("mergedSequences", jsonMergedSequences);        
4594 26 Sep 17 nicklas 130       }
4594 26 Sep 17 nicklas 131       else if ("GetUnconfirmedAlignedSequences".equals(cmd))
4594 26 Sep 17 nicklas 132       {
6335 15 Jun 21 nicklas 133         dc = sc.newDbControl(":Confirm Hisat alignment");
4594 26 Sep 17 nicklas 134         ItemQuery<DerivedBioAssay> query = DerivedBioAssay.getQuery();
4594 26 Sep 17 nicklas 135         query.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
4594 26 Sep 17 nicklas 136         Subtype.ALIGNED_SEQUENCES.addFilter(dc, query);
5546 07 Aug 19 nicklas 137         Pipeline.RNASEQ_HISAT_STRINGTIE.addFilter(dc, query);
4595 27 Sep 17 nicklas 138         // The job must be a HisatAlign job and it must be ended
4594 26 Sep 17 nicklas 139         query.join(Hql.innerJoin("job", "jb"));
4595 27 Sep 17 nicklas 140         Subtype.HISAT_ALIGN_JOB.addFilter(dc, query, "jb");
4594 26 Sep 17 nicklas 141         query.restrict(Restrictions.neq(Hql.property("jb", "ended"), null));
4594 26 Sep 17 nicklas 142         // Must NOT have a ANALYSIS_RESULT annotation
4594 26 Sep 17 nicklas 143         query.join(Annotations.leftJoin(null, Annotationtype.ANALYSIS_RESULT.load(dc), "ar"));
4594 26 Sep 17 nicklas 144         query.restrict(Restrictions.eq(Hql.alias("ar"), null));
4594 26 Sep 17 nicklas 145         // Ignore if AutoProcess==AutoConfirm
4594 26 Sep 17 nicklas 146         query.join(Annotations.leftJoin(null, Annotationtype.AUTO_PROCESSING.load(dc), "ap"));
4594 26 Sep 17 nicklas 147         query.restrict(
4594 26 Sep 17 nicklas 148             Restrictions.or(
4594 26 Sep 17 nicklas 149               Restrictions.neq(Hql.alias("ap"), Expressions.string("AutoConfirm")),
4594 26 Sep 17 nicklas 150               Restrictions.eq(Hql.alias("ap"), null)
4594 26 Sep 17 nicklas 151             ));
4677 08 Feb 18 nicklas 152         // Join LibPlate for sorting
4677 08 Feb 18 nicklas 153         query.join(Hql.innerJoin(null, "extract", "lib", true));
4677 08 Feb 18 nicklas 154         query.join(Hql.innerJoin("lib", "bioWell", "bw", true));
4677 08 Feb 18 nicklas 155         query.join(Hql.innerJoin("bw", "bioPlate", "libPlate", true));
4677 08 Feb 18 nicklas 156         query.order(Orders.asc(Hql.property("libPlate", "id")));
4594 26 Sep 17 nicklas 157         query.order(Orders.asc(Hql.property("name")));
4677 08 Feb 18 nicklas 158         query.setMaxResults(250);
4594 26 Sep 17 nicklas 159         
4594 26 Sep 17 nicklas 160         SnapshotManager manager = new SnapshotManager();
4594 26 Sep 17 nicklas 161         JSONArray jsonAligned = new JSONArray();
4594 26 Sep 17 nicklas 162         List<AlignedSequences> list = AlignedSequences.toList(query.list(dc));
4594 26 Sep 17 nicklas 163         for (AlignedSequences aligned : list)
4594 26 Sep 17 nicklas 164         {
4594 26 Sep 17 nicklas 165           aligned.loadAnnotations(dc, manager, "ALIGNED_PAIRS", Annotationtype.ALIGNED_PAIRS, null);
4594 26 Sep 17 nicklas 166           aligned.loadAnnotations(dc, manager, "READ_PAIRS_EXAMINED", Annotationtype.READ_PAIRS_EXAMINED, null);
4594 26 Sep 17 nicklas 167           aligned.loadAnnotations(dc, manager, "READ_PAIR_DUPLICATES", Annotationtype.READ_PAIR_DUPLICATES, null);
4594 26 Sep 17 nicklas 168           aligned.loadAnnotations(dc, manager, "FRACTION_DUPLICATION", Annotationtype.FRACTION_DUPLICATION, null);
4620 16 Nov 17 nicklas 169           aligned.loadAnnotations(dc, manager, "QC_GENOTYPE_COUNT", Annotationtype.QC_GENOTYPE_COUNT, null);
4658 26 Jan 18 nicklas 170           aligned.loadAnnotations(dc, manager, "QC_GENOTYPE_HET_PCT", Annotationtype.QC_GENOTYPE_HET_PCT, null);
4594 26 Sep 17 nicklas 171           aligned.loadAnnotations(dc, manager, "DataFilesFolder", Annotationtype.DATA_FILES_FOLDER, null);
4594 26 Sep 17 nicklas 172           
4620 16 Nov 17 nicklas 173           aligned.setAnnotation("qcVcf", JsonUtil.loadLinkedItem(dc, aligned.getItem(), "qc_genotype.vcf", Item.FILE, null));
4620 16 Nov 17 nicklas 174           
4594 26 Sep 17 nicklas 175           // Load job information
4594 26 Sep 17 nicklas 176           Job job = aligned.getItem().getJob();
4594 26 Sep 17 nicklas 177           JSONObject jsonJob = JsonUtil.getJobAsJSON(job);
4594 26 Sep 17 nicklas 178           aligned.setAnnotation("job", jsonJob);
4594 26 Sep 17 nicklas 179
4594 26 Sep 17 nicklas 180           boolean debug = Boolean.TRUE.equals(job.getParameterValue("debug"));
4594 26 Sep 17 nicklas 181           jsonJob.put("debug", debug);
4594 26 Sep 17 nicklas 182           
4594 26 Sep 17 nicklas 183           JSONObject jsonLimits = new JSONObject();
4600 28 Sep 17 nicklas 184           jsonLimits.put("minAlignedPairsFlag", debug ? HisatAlignAutoConfirmer.MIN_ALIGNED_PAIRS_FLAG_DEBUG : HisatAlignAutoConfirmer.MIN_ALIGNED_PAIRS_FLAG);
4600 28 Sep 17 nicklas 185           jsonLimits.put("minAlignedPairsStop", debug ? HisatAlignAutoConfirmer.MIN_ALIGNED_PAIRS_STOP_DEBUG : HisatAlignAutoConfirmer.MIN_ALIGNED_PAIRS_STOP);
4658 26 Jan 18 nicklas 186           jsonLimits.put("maxHetPct", HisatAlignAutoConfirmer.MAX_HET_PERCENTAGE);
4594 26 Sep 17 nicklas 187           jsonJob.put("limits", jsonLimits);
4594 26 Sep 17 nicklas 188           
4594 26 Sep 17 nicklas 189           // Load library information
4594 26 Sep 17 nicklas 190           Library lib = aligned.getLibrary(dc);
4594 26 Sep 17 nicklas 191           lib.loadBioPlateLocation();
4594 26 Sep 17 nicklas 192           aligned.setAnnotation("lib", lib.asJSONObject());
4594 26 Sep 17 nicklas 193           Rna r = lib.getRna(dc, true);
4594 26 Sep 17 nicklas 194           lib.setAnnotation("specimen", r.loadYellowLabelInfo(dc, manager));
4594 26 Sep 17 nicklas 195           
4656 24 Jan 18 nicklas 196           // Load masked information
4656 24 Jan 18 nicklas 197           MaskedSequences masked = aligned.getMaskedSequences(dc);
4685 23 Feb 18 nicklas 198           masked.loadAnnotations(dc, manager, "PM_READS", Annotationtype.PM_READS, null);
4685 23 Feb 18 nicklas 199           aligned.setAnnotation("masked", masked.asJSONObject());
4656 24 Jan 18 nicklas 200           
4594 26 Sep 17 nicklas 201           // Load merged information
4685 23 Feb 18 nicklas 202           MergedSequences merged = masked.getMergedSequences(dc);
4594 26 Sep 17 nicklas 203           merged.loadAnnotations(dc, manager, "PF_READS", Annotationtype.PF_READS, null);
4594 26 Sep 17 nicklas 204           merged.loadAnnotations(dc, manager, "PT_READS", Annotationtype.PT_READS, null);
4594 26 Sep 17 nicklas 205           aligned.setAnnotation("merged", merged.asJSONObject());
4594 26 Sep 17 nicklas 206           
4594 26 Sep 17 nicklas 207           jsonAligned.add(aligned.asJSONObject());
4594 26 Sep 17 nicklas 208         }
4594 26 Sep 17 nicklas 209         
4594 26 Sep 17 nicklas 210         json.put("alignedSequences", jsonAligned);
4594 26 Sep 17 nicklas 211         
4594 26 Sep 17 nicklas 212       }
4594 26 Sep 17 nicklas 213     }
4594 26 Sep 17 nicklas 214     catch (Throwable t)
4594 26 Sep 17 nicklas 215     {
4594 26 Sep 17 nicklas 216       t.printStackTrace();
4594 26 Sep 17 nicklas 217       json.clear();
4594 26 Sep 17 nicklas 218       json.put("status", "error");
4594 26 Sep 17 nicklas 219       json.put("message", t.getMessage());
4594 26 Sep 17 nicklas 220       json.put("stacktrace", ThrowableUtil.stackTraceToString(t));
4594 26 Sep 17 nicklas 221     }
4594 26 Sep 17 nicklas 222     finally
4594 26 Sep 17 nicklas 223     {
4594 26 Sep 17 nicklas 224       if (dc != null) dc.close();
4594 26 Sep 17 nicklas 225       json.writeJSONString(resp.getWriter());
4594 26 Sep 17 nicklas 226     }
4594 26 Sep 17 nicklas 227     
4594 26 Sep 17 nicklas 228   }
4594 26 Sep 17 nicklas 229
4594 26 Sep 17 nicklas 230   @Override
4594 26 Sep 17 nicklas 231   protected void doPost(HttpServletRequest req, HttpServletResponse resp)
4594 26 Sep 17 nicklas 232     throws ServletException, IOException 
4594 26 Sep 17 nicklas 233   {
4594 26 Sep 17 nicklas 234     String cmd = req.getParameter("cmd");
4594 26 Sep 17 nicklas 235     JsonUtil.setJsonResponseHeaders(resp);
4594 26 Sep 17 nicklas 236     
4594 26 Sep 17 nicklas 237     JSONObject json = new JSONObject();
4594 26 Sep 17 nicklas 238     json.put("status", "ok");
4594 26 Sep 17 nicklas 239     JSONArray jsonMessages = new JSONArray();
4594 26 Sep 17 nicklas 240   
4594 26 Sep 17 nicklas 241     final SessionControl sc = Reggie.getSessionControl(req);
4594 26 Sep 17 nicklas 242     DbControl dc = null;
4594 26 Sep 17 nicklas 243     try
4594 26 Sep 17 nicklas 244     {
4594 26 Sep 17 nicklas 245
4594 26 Sep 17 nicklas 246       if ("StartHisatAlignment".equals(cmd))
4594 26 Sep 17 nicklas 247       {
6335 15 Jun 21 nicklas 248         dc = sc.newDbControl(":Start Hisat alignment");
4594 26 Sep 17 nicklas 249
4594 26 Sep 17 nicklas 250         ReggieRole.checkPermission(dc, "'" + cmd + "' wizard", ReggieRole.SECONDARY_ANALYSIS, ReggieRole.ADMINISTRATOR);
4594 26 Sep 17 nicklas 251
4594 26 Sep 17 nicklas 252         JSONObject jsonReq = JsonUtil.parseRequest(req);
4594 26 Sep 17 nicklas 253         JSONArray jsonMerged = (JSONArray)jsonReq.get("mergedSequences");
4594 26 Sep 17 nicklas 254         
4656 24 Jan 18 nicklas 255         Number maskingSoftwareId = (Number)jsonReq.get("maskingSoftware");
4656 24 Jan 18 nicklas 256         Number maskingProtocolId = (Number)jsonReq.get("maskingProtocol");
4594 26 Sep 17 nicklas 257         Number alignSoftwareId = (Number)jsonReq.get("alignSoftware");
4594 26 Sep 17 nicklas 258         Number alignProtocolId = (Number)jsonReq.get("alignProtocol");
4594 26 Sep 17 nicklas 259         
4594 26 Sep 17 nicklas 260         String clusterId = (String)jsonReq.get("cluster");
4594 26 Sep 17 nicklas 261         boolean debug = Boolean.TRUE.equals(jsonReq.get("debug"));
4594 26 Sep 17 nicklas 262         boolean autoConfirm = Boolean.TRUE.equals(jsonReq.get("autoConfirm"));
4594 26 Sep 17 nicklas 263         Number priority = (Number)jsonReq.get("priority");
6981 17 Jan 23 nicklas 264         String partition = Values.getStringOrNull((String)jsonReq.get("partition"));
7372 06 Oct 23 nicklas 265         String submitOptions = Values.getStringOrNull((String)jsonReq.get("submitOptionsOverride"));
4594 26 Sep 17 nicklas 266
4594 26 Sep 17 nicklas 267         OpenGridCluster cluster = OpenGridService.getInstance().getClusterById(dc, clusterId);
4594 26 Sep 17 nicklas 268         if (cluster == null)
4594 26 Sep 17 nicklas 269         {
4594 26 Sep 17 nicklas 270           throw new ItemNotFoundException("OpenGridCluster[" + clusterId + "]");
4594 26 Sep 17 nicklas 271         }
4594 26 Sep 17 nicklas 272         
4594 26 Sep 17 nicklas 273         // Load common items
4656 24 Jan 18 nicklas 274         Software maskingSoftware = maskingSoftwareId != null ? Software.getById(dc, maskingSoftwareId.intValue()) : null;
4656 24 Jan 18 nicklas 275         Protocol maskingProtocol = maskingProtocolId != null ? Protocol.getById(dc, maskingProtocolId.intValue()) : null;
4594 26 Sep 17 nicklas 276         Software alignSoftware = alignSoftwareId != null ? Software.getById(dc, alignSoftwareId.intValue()) : null;
4594 26 Sep 17 nicklas 277         Protocol alignProtocol = alignProtocolId != null ? Protocol.getById(dc, alignProtocolId.intValue()) : null;
4594 26 Sep 17 nicklas 278
4594 26 Sep 17 nicklas 279         // Align job creator implementation
4594 26 Sep 17 nicklas 280         HisatAlignJobCreator jobCreator = new HisatAlignJobCreator();
4594 26 Sep 17 nicklas 281         jobCreator.setAutoConfirm(autoConfirm);
4594 26 Sep 17 nicklas 282         jobCreator.setDebug(debug);
4594 26 Sep 17 nicklas 283         jobCreator.setPriority(priority == null ? null : priority.intValue());
6981 17 Jan 23 nicklas 284         jobCreator.setPartition(partition);
4656 24 Jan 18 nicklas 285         jobCreator.setMaskingProtocol(maskingProtocol);
4656 24 Jan 18 nicklas 286         jobCreator.setMaskingSoftware(maskingSoftware);
4594 26 Sep 17 nicklas 287         jobCreator.setAlignProtocol(alignProtocol);
4594 26 Sep 17 nicklas 288         jobCreator.setAlignSoftware(alignSoftware);
7372 06 Oct 23 nicklas 289         jobCreator.setSubmitOptionsOverride(submitOptions);
4594 26 Sep 17 nicklas 290
4594 26 Sep 17 nicklas 291         List<MergedSequences> mergedSequences = new ArrayList<MergedSequences>();
4594 26 Sep 17 nicklas 292         for (int mgNo = 0; mgNo < jsonMerged.size(); mgNo++)
4594 26 Sep 17 nicklas 293         {
4594 26 Sep 17 nicklas 294           JSONObject jsonMg = (JSONObject)jsonMerged.get(mgNo);
4594 26 Sep 17 nicklas 295           Number mergedId = (Number)jsonMg.get("id");
4594 26 Sep 17 nicklas 296         
4594 26 Sep 17 nicklas 297           MergedSequences ms = MergedSequences.getById(dc, mergedId.intValue());
4594 26 Sep 17 nicklas 298           mergedSequences.add(ms);
4594 26 Sep 17 nicklas 299
4594 26 Sep 17 nicklas 300           // Reset AUTO_PROCESSING annotation
4594 26 Sep 17 nicklas 301           DerivedBioAssay merged = ms.getDerivedBioAssay();
4594 26 Sep 17 nicklas 302           Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, merged, null);
4594 26 Sep 17 nicklas 303         }
4594 26 Sep 17 nicklas 304         
4599 28 Sep 17 nicklas 305         List<JobDefinition> jobDefs = jobCreator.createHisatAlignJobs(dc, cluster, mergedSequences);
4599 28 Sep 17 nicklas 306         List<Job> jobs = ScriptUtil.submitJobs(dc, cluster, jobDefs);
4594 26 Sep 17 nicklas 307         for (Job job : jobs)
4594 26 Sep 17 nicklas 308         {
4594 26 Sep 17 nicklas 309           if (job.getStatus() == Job.Status.ERROR)
4594 26 Sep 17 nicklas 310           {
4594 26 Sep 17 nicklas 311             jsonMessages.add("[Error]Job submission for '" + job.getName() + "' failed: " + job.getStatusMessage());
4594 26 Sep 17 nicklas 312           }
4594 26 Sep 17 nicklas 313           else
4594 26 Sep 17 nicklas 314           {
4594 26 Sep 17 nicklas 315             jsonMessages.add("Submitted Hisat alignment job to " + cluster.getConnectionInfo().getName() + " with id " + job.getExternalId());
4594 26 Sep 17 nicklas 316           }
4594 26 Sep 17 nicklas 317         }
4594 26 Sep 17 nicklas 318         dc.commit();
4594 26 Sep 17 nicklas 319       }
4595 27 Sep 17 nicklas 320       else if ("RegisterHisatAlignment".equals(cmd))
4594 26 Sep 17 nicklas 321       {
6335 15 Jun 21 nicklas 322         dc = sc.newDbControl(":Confirm Hisat alignment");
4594 26 Sep 17 nicklas 323
4594 26 Sep 17 nicklas 324         ReggieRole.checkPermission(dc, "'" + cmd + "' wizard", ReggieRole.SECONDARY_ANALYSIS, ReggieRole.ADMINISTRATOR);
4594 26 Sep 17 nicklas 325
4594 26 Sep 17 nicklas 326         JSONObject jsonReq = JsonUtil.parseRequest(req);
4594 26 Sep 17 nicklas 327         JSONArray jsonAlignedSequences = (JSONArray)jsonReq.get("alignedSequences");
4594 26 Sep 17 nicklas 328
4594 26 Sep 17 nicklas 329         int numConfirmed = 0;
4594 26 Sep 17 nicklas 330         int numUnconfirmed = 0;
4594 26 Sep 17 nicklas 331         int numRealign = 0;
4594 26 Sep 17 nicklas 332         int numFlaggedRna = 0;
4594 26 Sep 17 nicklas 333         int numDeleted = 0;
4594 26 Sep 17 nicklas 334         
4594 26 Sep 17 nicklas 335         ItemList flaggedRna = BiomaterialList.FLAGGED_RNA.load(dc);
4598 27 Sep 17 nicklas 336         ItemList hisatPipeline = BiomaterialList.HISAT_PIPELINE.load(dc);
4660 29 Jan 18 nicklas 337         ItemList stringtiePipeline = BiomaterialList.STRINGTIE_PIPELINE.load(dc);
5015 08 Oct 18 nicklas 338         ItemList mbafPipeline = BiomaterialList.MBAF_PIPELINE.load(dc);
5683 22 Oct 19 nicklas 339         ItemList variantCallingPipeline = BiomaterialList.VARIANT_CALLING_PIPELINE.load(dc);
4594 26 Sep 17 nicklas 340         
4594 26 Sep 17 nicklas 341         boolean deleteItemsCreatedByFailedJobs = Boolean.TRUE.equals(jsonReq.get("deleteItemsCreatedByFailedJobs"));
4594 26 Sep 17 nicklas 342         List<BasicItem> toDelete = new ArrayList<BasicItem>();
4594 26 Sep 17 nicklas 343         
4594 26 Sep 17 nicklas 344         for (int alignNo = 0; alignNo < jsonAlignedSequences.size(); ++alignNo)
4594 26 Sep 17 nicklas 345         {
4594 26 Sep 17 nicklas 346           JSONObject jsonAligned = (JSONObject)jsonAlignedSequences.get(alignNo);
4594 26 Sep 17 nicklas 347           Number alignedId = (Number)jsonAligned.get("id");
4594 26 Sep 17 nicklas 348           
4594 26 Sep 17 nicklas 349           AlignedSequences aligned = AlignedSequences.getById(dc, alignedId.intValue());
4656 24 Jan 18 nicklas 350           MaskedSequences masked = aligned.getMaskedSequences(dc);
4685 23 Feb 18 nicklas 351           MergedSequences merged = masked.getMergedSequences(dc);
4594 26 Sep 17 nicklas 352           
4594 26 Sep 17 nicklas 353           DerivedBioAssay alignedDBA = aligned.getDerivedBioAssay();
4594 26 Sep 17 nicklas 354           Job alignJob = alignedDBA.getJob();
4594 26 Sep 17 nicklas 355           Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, alignedDBA, null);
4594 26 Sep 17 nicklas 356           
4658 26 Jan 18 nicklas 357           String flag = (String)(jsonAligned.get("flag"));
5553 12 Aug 19 nicklas 358           if (flag != null && !Reggie.isExternalItem(aligned.getName()))
4594 26 Sep 17 nicklas 359           {
4594 26 Sep 17 nicklas 360             Library lib = Library.get(alignedDBA.getExtract());
4594 26 Sep 17 nicklas 361             Rna rna = lib.getRna(dc, false);
4594 26 Sep 17 nicklas 362             Extract r = rna.getItem();
4658 26 Jan 18 nicklas 363             Annotationtype.FLAG.setAnnotationValue(dc, r, flag);
4594 26 Sep 17 nicklas 364             flaggedRna.add(r);
4594 26 Sep 17 nicklas 365             numFlaggedRna++;
4594 26 Sep 17 nicklas 366           }
4594 26 Sep 17 nicklas 367           
4594 26 Sep 17 nicklas 368           String comment = Values.getStringOrNull((String)jsonAligned.get("comment"));
4594 26 Sep 17 nicklas 369           alignedDBA.setDescription(comment);
4594 26 Sep 17 nicklas 370           
4594 26 Sep 17 nicklas 371           boolean reAlign = Boolean.TRUE.equals(jsonAligned.get("realign"));
4663 29 Jan 18 nicklas 372           boolean startStringTie = Boolean.TRUE.equals(jsonAligned.get("stringtie"));
4594 26 Sep 17 nicklas 373           boolean deleted = false;
4594 26 Sep 17 nicklas 374
4594 26 Sep 17 nicklas 375           if (reAlign)
4594 26 Sep 17 nicklas 376           {
4594 26 Sep 17 nicklas 377             // Set ReProcess annotation on merged sequences item
4594 26 Sep 17 nicklas 378             Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, merged.getItem(), "ReProcess");
4598 27 Sep 17 nicklas 379             hisatPipeline.add(merged.getItem());
4594 26 Sep 17 nicklas 380             numRealign++;
4594 26 Sep 17 nicklas 381             
4594 26 Sep 17 nicklas 382             if (deleteItemsCreatedByFailedJobs && alignJob.getStatus() == Job.Status.ERROR)
4594 26 Sep 17 nicklas 383             {
4595 27 Sep 17 nicklas 384               // Delete aligned sequences items
4594 26 Sep 17 nicklas 385               alignedDBA.setRemoved(true);
4594 26 Sep 17 nicklas 386               toDelete.add(alignedDBA);
4594 26 Sep 17 nicklas 387               toDelete.addAll(Reggie.removeAttachedFiles(dc, alignedDBA));
4685 23 Feb 18 nicklas 388               
4685 23 Feb 18 nicklas 389               DerivedBioAssay mask = masked.getItem();
4685 23 Feb 18 nicklas 390               mask.setRemoved(true);
4685 23 Feb 18 nicklas 391               toDelete.add(mask);
4685 23 Feb 18 nicklas 392               toDelete.addAll(Reggie.removeAttachedFiles(dc, mask));
4685 23 Feb 18 nicklas 393
4594 26 Sep 17 nicklas 394               deleted = true;
4594 26 Sep 17 nicklas 395               numDeleted++;
4595 27 Sep 17 nicklas 396               if (comment != null)
4595 27 Sep 17 nicklas 397               {
4595 27 Sep 17 nicklas 398                 merged.getItem().setDescription(comment);
4595 27 Sep 17 nicklas 399               }
4594 26 Sep 17 nicklas 400             }
4594 26 Sep 17 nicklas 401           }
4594 26 Sep 17 nicklas 402           
4594 26 Sep 17 nicklas 403           if (!deleted)
4594 26 Sep 17 nicklas 404           {
4663 29 Jan 18 nicklas 405             if (startStringTie)
4594 26 Sep 17 nicklas 406             {
4594 26 Sep 17 nicklas 407               Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, alignedDBA, AlignedSequences.ALIGN_SUCCESSFUL);
4660 29 Jan 18 nicklas 408               stringtiePipeline.add(alignedDBA);
5015 08 Oct 18 nicklas 409               mbafPipeline.add(alignedDBA);
5683 22 Oct 19 nicklas 410               variantCallingPipeline.add(alignedDBA);
4594 26 Sep 17 nicklas 411               numConfirmed++;
4594 26 Sep 17 nicklas 412             }
4658 26 Jan 18 nicklas 413             else if (reAlign || flag != null)
4594 26 Sep 17 nicklas 414             {
4594 26 Sep 17 nicklas 415               Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, alignedDBA, AlignedSequences.ALIGN_FAILED);
4594 26 Sep 17 nicklas 416             }
4594 26 Sep 17 nicklas 417             else
4594 26 Sep 17 nicklas 418             {
4594 26 Sep 17 nicklas 419               numUnconfirmed++;
4594 26 Sep 17 nicklas 420             }
4594 26 Sep 17 nicklas 421           }
4594 26 Sep 17 nicklas 422         }
5845 26 Feb 20 nicklas 423         
4594 26 Sep 17 nicklas 424         if (numConfirmed > 0)
4594 26 Sep 17 nicklas 425         {
4663 29 Jan 18 nicklas 426           jsonMessages.add(numConfirmed + " libraries confirmed for StringTie");
5421 13 May 19 nicklas 427           ActivityDef.HISAT_CONFIRMED.merge(dc, numConfirmed);
4594 26 Sep 17 nicklas 428         }
4594 26 Sep 17 nicklas 429         else
4594 26 Sep 17 nicklas 430         {
4594 26 Sep 17 nicklas 431           jsonMessages.add("No libraries confirmed as aligned");
4594 26 Sep 17 nicklas 432         }
4594 26 Sep 17 nicklas 433         
4594 26 Sep 17 nicklas 434         if (numRealign > 0)
4594 26 Sep 17 nicklas 435         {
4594 26 Sep 17 nicklas 436           jsonMessages.add(numRealign + " libraries flagged for re-alignment");
4594 26 Sep 17 nicklas 437         }
4594 26 Sep 17 nicklas 438         
4594 26 Sep 17 nicklas 439         if (numDeleted > 0)
4594 26 Sep 17 nicklas 440         {
4594 26 Sep 17 nicklas 441           jsonMessages.add(numDeleted + " libraries deleted due to failure");
4594 26 Sep 17 nicklas 442           if (toDelete.size() > numDeleted)
4594 26 Sep 17 nicklas 443           {
4594 26 Sep 17 nicklas 444             jsonMessages.add((toDelete.size() - numDeleted) + " linked items (eg. files) deleted");
4594 26 Sep 17 nicklas 445           }
4594 26 Sep 17 nicklas 446         }
4594 26 Sep 17 nicklas 447         
4594 26 Sep 17 nicklas 448         if (numFlaggedRna > 0)
4594 26 Sep 17 nicklas 449         {
4594 26 Sep 17 nicklas 450           jsonMessages.add("Added " + numFlaggedRna + " RNA items to the '" + flaggedRna.getName() + "' list");
4594 26 Sep 17 nicklas 451         }
4594 26 Sep 17 nicklas 452         
4594 26 Sep 17 nicklas 453         if (numUnconfirmed > 0)
4594 26 Sep 17 nicklas 454         {
4594 26 Sep 17 nicklas 455           jsonMessages.add(numUnconfirmed + " libraries remain unconfirmed");
4594 26 Sep 17 nicklas 456         }
4594 26 Sep 17 nicklas 457         
4594 26 Sep 17 nicklas 458         dc.commit();
4594 26 Sep 17 nicklas 459         
4594 26 Sep 17 nicklas 460         if (toDelete.size() > 0)
4594 26 Sep 17 nicklas 461         {
4594 26 Sep 17 nicklas 462           try
4594 26 Sep 17 nicklas 463           {
4594 26 Sep 17 nicklas 464             Trashcan.delete(sc, toDelete, false, null);
4594 26 Sep 17 nicklas 465           }
4594 26 Sep 17 nicklas 466           catch (RuntimeException ex)
4594 26 Sep 17 nicklas 467           {
4594 26 Sep 17 nicklas 468             ex.printStackTrace();
4594 26 Sep 17 nicklas 469             jsonMessages.add("[Error]Could not delete all items created by failed jobs: " + ex.getMessage());
4594 26 Sep 17 nicklas 470           }
4594 26 Sep 17 nicklas 471         }
4594 26 Sep 17 nicklas 472       }
4594 26 Sep 17 nicklas 473
4594 26 Sep 17 nicklas 474       json.put("messages", jsonMessages);
4594 26 Sep 17 nicklas 475       CounterService.getInstance().setForceCount();
4594 26 Sep 17 nicklas 476     }
4594 26 Sep 17 nicklas 477     catch (Throwable t)
4594 26 Sep 17 nicklas 478     {
4594 26 Sep 17 nicklas 479       t.printStackTrace();
4594 26 Sep 17 nicklas 480       json.clear();
4594 26 Sep 17 nicklas 481       json.put("status", "error");
4594 26 Sep 17 nicklas 482       json.put("message", t.getMessage());
4594 26 Sep 17 nicklas 483       json.put("stacktrace", ThrowableUtil.stackTraceToString(t));
4594 26 Sep 17 nicklas 484     }
4594 26 Sep 17 nicklas 485     finally
4594 26 Sep 17 nicklas 486     {
4594 26 Sep 17 nicklas 487       if (dc != null) dc.close();
4594 26 Sep 17 nicklas 488       json.writeJSONString(resp.getWriter());
4594 26 Sep 17 nicklas 489     }
4594 26 Sep 17 nicklas 490     
4594 26 Sep 17 nicklas 491   }
4594 26 Sep 17 nicklas 492   
4594 26 Sep 17 nicklas 493   
4594 26 Sep 17 nicklas 494
4594 26 Sep 17 nicklas 495 }

Generated on Tue Nov 21 03:35:20 2023 (UTC) by svndigest 0.11.2pre (dev build 1672)