dials.integrate

Introduction

This program is used to integrate the reflections on the diffraction images. It is called with an experiment list outputted from dials.index or dials.refine and a corresponding set of strong spots from which a profile model is calculated. The program will output a set of integrated reflections and an experiment list with additional profile model data. The data can be reintegrated using the same profile model by inputting this integrated_experiments.json file back into dials.integate.

Examples:

dials.integrate experiments.json indexed.pickle

dials.integrate experiments.json indexed.pickle output.reflections=integrated.pickle

dials.integrate experiments.json indexed.pickle profile.fitting=False

dials.integrate experiments.json indexed.pickle background.algorithm=glm

Basic parameters

output {
  experiments = 'integrated_experiments.json'
  reflections = 'integrated.pickle'
  phil = 'dials.integrate.phil'
  log = 'dials.integrate.log'
  debug_log = 'dials.integrate.debug.log'
  report = None
  include_bad_reference = False
}
scan_range = None
create_profile_model = True
exclude_images = None
verbosity = 1
integration {
  lookup {
    mask = None
  }
  block {
    size = auto
    units = *degrees radians frames
    threshold = 0.99
    force = False
    max_memory_usage = 0.75
  }
  use_dynamic_mask = True
  debug {
    reference {
      filename = "reference_profiles.pickle"
      output = False
    }
    during = modelling *integration
    output = False
    separate_files = True
    delete_shoeboxes = False
    select = None
    split_experiments = True
  }
  profile {
    fitting = True
    validation {
      number_of_partitions = 1
      min_partition_size = 100
    }
  }
  overlaps_filter {
    foreground_foreground {
      enable = False
    }
    foreground_background {
      enable = False
    }
  }
  mp {
    method = *none drmaa sge lsf pbs
    njobs = 1
    nproc = 1
  }
  summation {
    detector_gain = 1
  }
  background {
    algorithm = simple null median gmodel *glm
    simple {
      outlier {
        algorithm = *null nsigma truncated normal plane tukey
      }
      model {
        algorithm = constant2d *constant3d linear2d linear3d
      }
      min_pixels = 10
    }
    median {
    }
    gmodel {
      robust {
        algorithm = False
        tuning_constant = 1.345
      }
      min_pixels = 10
      model = None
    }
    glm {
      robust {
        tuning_constant = 1.345
      }
      model {
        algorithm = constant2d *constant3d loglinear2d loglinear3d
      }
      min_pixels = 10
    }
  }
  centroid {
    algorithm = *simple
  }
}
profile {
  algorithm = gaussian_rs *gaussian_rs
  gaussian_rs {
    scan_varying = False
    min_spots {
      overall = 50
      per_degree = 20
    }
    sigma_m_algorithm = basic *extended
    parameters {
      sigma_b = None
      sigma_m = None
    }
    filter {
      min_zeta = 0.05
    }
    fitting {
      scan_step = 5
      grid_size = 5
      threshold = 0.02
      grid_method = single *regular_grid circular_grid spherical_grid
      fit_method = *reciprocal_space detector_space
      detector_space {
        deconvolution = False
      }
    }
  }
  gaussian_rs {
    scan_varying = False
    min_spots {
      overall = 50
      per_degree = 20
    }
    sigma_m_algorithm = basic *extended
    parameters {
      sigma_b = None
      sigma_m = None
    }
    filter {
      min_zeta = 0.05
    }
    fitting {
      scan_step = 5
      grid_size = 5
      threshold = 0.02
      grid_method = single *regular_grid circular_grid spherical_grid
      fit_method = *reciprocal_space detector_space
      detector_space {
        deconvolution = False
      }
    }
  }
}
prediction {
  d_min = None
  d_max = None
  margin = 1
  force_static = False
  padding = 1.0
}
significance_filter {
  enable = False
  n_bins = 20
  isigi_cutoff = 1.0
}
absorption_correction {
  apply = False
  algorithm = fuller_kapton other
  fuller_kapton {
    xtal_height_above_kapton_mm {
      value = 0.02
      sigma = 0.01
    }
    rotation_angle_deg {
      value = 1.15
      sigma = 0.1
    }
    kapton_half_width_mm {
      value = 1.5875
      sigma = 0.5
    }
    kapton_thickness_mm {
      value = 0.05
      sigma = 0.005
    }
    smart_sigmas = False
    within_spot_sigmas = True
  }
}

Full parameter definitions

output {
  experiments = 'integrated_experiments.json'
    .help = "The experiments output filename"
    .type = str
  reflections = 'integrated.pickle'
    .help = "The integrated output filename"
    .type = str
  phil = 'dials.integrate.phil'
    .help = "The output phil file"
    .type = str
  log = 'dials.integrate.log'
    .help = "The log filename"
    .type = str
  debug_log = 'dials.integrate.debug.log'
    .help = "The debug log filename"
    .type = str
  report = None
    .help = "The integration report filename (*.xml or *.json)"
    .type = str
  include_bad_reference = False
    .help = "Include bad reference data including unindexed spots, and"
            "reflections whose predictions are messed up in the reflection"
            "table output. Reflections will have the 'bad_reference' flag set."
    .type = bool
}
scan_range = None
  .help = "Explicitly specify the images to be processed. Only applicable when"
          "experiment list contains a single imageset."
  .type = ints(size=2)
  .multiple = True
create_profile_model = True
  .help = "Create the profile model"
  .type = bool
sampling
  .expert_level = 1
{
  reflections_per_degree = 50
    .help = "The number of predicted reflections per degree of the sweep  to"
            "integrate."
    .type = float(value_min=0)
  minimum_sample_size = 1000
    .help = "cutoff that determines whether subsetting of the input "
            "prediction list is done"
    .type = int
  maximum_sample_size = None
    .help = "The maximum number of predictions to integrate. Overrides"
            "reflections_per_degree if that produces a larger sample size."
    .type = int(value_min=1)
  integrate_all_reflections = True
    .help = "Override reflections_per_degree and integrate all predicted"
            "reflections."
    .type = bool
}
exclude_images = None
  .help = "Exclude images from integration (e.g. 1,2,3,4,5 etc)"
  .type = ints
verbosity = 1
  .help = "The verbosity level"
  .type = int(value_min=0)
integration {
  lookup
    .help = "Parameters specifying lookup file path"
  {
    mask = None
      .help = "The path to the mask file."
      .type = str
  }
  block {
    size = auto
      .help = "The block size in rotation angle (degrees)."
      .type = float
    units = *degrees radians frames
      .help = "The units of the block size"
      .type = choice
    threshold = 0.99
      .help = "For block size auto the block size is calculated by sorting"
              "reflections by the number of frames they cover and then"
              "selecting the block size to be 2*nframes[threshold] such that"
              "100*threshold % of reflections are guarenteed to be fully"
              "contained in 1 block"
      .type = float(value_min=0, value_max=1)
    force = False
      .help = "If the number of processors is 1 and force is False, then the"
              "number of blocks may be set to 1. If force is True then the"
              "block size is always calculated."
      .type = bool
    max_memory_usage = 0.75
      .help = "The maximum percentage of total physical memory to use for"
              "allocating shoebox arrays."
      .type = float(value_min=0, value_max=1)
  }
  use_dynamic_mask = True
    .help = "Use dynamic mask if available"
    .type = bool
  debug {
    reference {
      filename = "reference_profiles.pickle"
        .help = "The filename for the reference profiles"
        .type = str
      output = False
        .help = "Save the reference profiles"
        .type = bool
    }
    during = modelling *integration
      .help = "Do debugging during modelling or integration"
      .type = choice
    output = False
      .help = "Save shoeboxes after each processing task."
      .type = bool
    separate_files = True
      .help = "If this is true, the shoeboxes are saved in separate files from"
              "the output integrated.pickle file. This is necessary in most"
              "cases since the amount of memory used by the shoeboxes is"
              "typically greater than the available system memory. If,"
              "however, you know that memory is not an issue, you can saved"
              "the shoeboxes in the integrated.pickle file by setting this"
              "option to False. This only works if the debug output is during"
              "integrated and not modelling."
      .type = bool
    delete_shoeboxes = False
      .help = "Delete shoeboxes immediately before saving files. This option"
              "in combination with debug.output=True enables intermediate"
              "processing steps to make use of shoeboxes."
      .type = bool
    select = None
      .help = "A string specifying the selection. The string should be of the"
              "form: select=${COLUMN}[<|<=|==|!=|>=|>]${VALUE}. In addition to"
              "the items in the reflection table, the following implicit"
              "columns are defined if the necessary data is there: "
              "intensity.sum.i_over_sigma  intensity.prf.i_over_sigma"
      .type = reflection_table_selector
    split_experiments = True
      .help = "Split shoeboxes into different files"
      .type = bool
  }
  profile {
    fitting = True
      .help = "Use profile fitting if available"
      .type = bool
    validation {
      number_of_partitions = 1
        .help = "The number of subsamples to take from the reference spots. If"
                "the value is 1, then no validation is performed."
        .type = int(value_min=1)
      min_partition_size = 100
        .help = "The minimum number of spots to use in each subsample."
        .type = int(value_min=1)
    }
  }
  filter
    .expert_level = 1
  {
    min_zeta = 0.05
      .help = "Filter the reflections by the value of zeta. A value of less"
              "than or equal to zero indicates that this will not be used. A"
              "positive value is used as the minimum permissable value."
      .type = float(value_min=0, value_max=1)
    max_shoebox_overlap = 1.0
      .help = "Filter reflections whose shoeboxes are overlapped by greater"
              "than the requested amount. Note that this is not the percentage"
              "of the peak that is overlapped but rather the percentage of the"
              "shoebox (background and foreground). This can be useful when"
              "the detector is too close and many overlapping reflections are"
              "predicted at high resolution causing memory issues."
      .type = float(value_min=0, value_max=1)
    ice_rings = False
      .help = "Set the ice ring flags"
      .type = bool
  }
  overlaps_filter {
    foreground_foreground {
      enable = False
        .help = "Remove all spots in which neighbors' foreground impinges on"
                "the spot's foreground"
        .type = bool
    }
    foreground_background {
      enable = False
        .help = "Remove all spots in which neighbors' foreground impinges on"
                "the spot's background"
        .type = bool
    }
  }
  mp {
    method = *none drmaa sge lsf pbs
      .help = "The cluster method to use"
      .type = choice
    njobs = 1
      .help = "The number of cluster jobs to use"
      .type = int(value_min=1)
    nproc = 1
      .help = "The number of processes to use per cluster job"
      .type = int(value_min=1)
  }
  summation {
    detector_gain = 1
      .help = "Multiplier for variances after integration of still images. See"
              "Leslie 1999."
      .type = float
  }
  background
    .help = "  I n t e r f a c e   f o r   b a c k g r o u n d   a l g o r i t"
            "h m s .  "
  {
    algorithm = simple null median gmodel *glm
      .help = "The choice of algorithm"
      .type = choice
    simple
      .help = "An extension class implementing simple background subtraction."
    {
      outlier
        .help = "Outlier rejection prior to background fit"
      {
        algorithm = *null nsigma truncated normal plane tukey
          .help = "The outlier rejection algorithm."
          .type = choice
        nsigma
          .help = "Parameters for nsigma outlier rejector"
          .expert_level = 1
        {
          lower = 3
            .help = "Lower n sigma"
            .type = float
          upper = 3
            .help = "Upper n sigma"
            .type = float
        }
        truncated
          .help = "Parameters for truncated outlier rejector"
          .expert_level = 1
        {
          lower = 0.01
            .help = "Lower bound"
            .type = float
          upper = 0.01
            .help = "Upper bound"
            .type = float
        }
        normal
          .help = "Parameters for normal outlier rejector"
          .expert_level = 1
        {
          min_pixels = 10
            .help = "The minimum number of pixels to use in calculating the"
                    "background intensity."
            .type = int
        }
        plane
          .help = "Parameters for mosflm-like outlier rejector. This algorithm"
                  "is mainly used in conjunction with a linear 2d background."
          .expert_level = 1
        {
          fraction = 1.0
            .help = "The fraction of pixels to use in determining the initial"
                    "plane used for outlier rejection."
            .type = float
          n_sigma = 4.0
            .help = "The number of standard deviations above the threshold"
                    "plane to use in rejecting outliers from background"
                    "calculation."
            .type = float
        }
        tukey
          .help = "Parameters for tukey outlier rejector"
          .expert_level = 1
        {
          lower = 1.5
            .help = "Lower IQR multiplier"
            .type = float
          upper = 1.5
            .help = "Upper IQR multiplier"
            .type = float
        }
      }
      model
        .help = "Background model"
      {
        algorithm = constant2d *constant3d linear2d linear3d
          .help = "The choice of background model"
          .type = choice
      }
      min_pixels = 10
        .help = "The minimum number of pixels to compute the background"
        .type = int(value_min=1)
    }
    median
      .help = "An extension class implementing a median background algorithm."
    {
    }
    gmodel
      .help = "An extension class implementing a robust GLM background"
              "algorithm."
    {
      robust {
        algorithm = False
          .help = "Use the robust algorithm"
          .type = bool
        tuning_constant = 1.345
          .help = "The tuning constant for robust estimation"
          .type = float
      }
      min_pixels = 10
        .help = "The minimum number of pixels required"
        .type = int(value_min=1)
      model = None
        .help = "The model filename"
        .type = str
    }
    glm
      .help = "An extension class implementing a robust GLM background"
              "algorithm."
    {
      robust {
        tuning_constant = 1.345
          .help = "The tuning constant for robust estimation"
          .type = float
      }
      model {
        algorithm = constant2d *constant3d loglinear2d loglinear3d
          .help = "The background model to fit"
          .type = choice
      }
      min_pixels = 10
        .help = "The minimum number of pixels required"
        .type = int(value_min=1)
    }
  }
  centroid
    .help = "  I n t e r f a c e   f o r   c e n t r o i d   a l g o r i t h m"
            "s .  "
  {
    algorithm = *simple
      .help = "The choice of algorithm"
      .type = choice
  }
}
profile
  .help = "
     T h e   i n t e r f a c e   d e f i n i t i o n   f o r   a  "
          "p r o f i l e   m o d e l . 
 
    "
{
  algorithm = gaussian_rs *gaussian_rs
    .help = "The choice of algorithm"
    .type = choice
  gaussian_rs
    .help = "An extension class implementing a reciprocal space gaussian"
            "profile model."
  {
    scan_varying = False
      .help = "Calculate a scan varying model"
      .type = bool
    min_spots
      .help = "if (total_reflections > overall or reflections_per_degree >"
              "per_degree) then do the profile modelling."
    {
      overall = 50
        .help = "The minimum number of spots needed to do the profile"
                "modelling"
        .type = int(value_min=0)
      per_degree = 20
        .help = "The minimum number of spots needed to do the profile"
                "modelling"
        .type = int(value_min=0)
    }
    sigma_m_algorithm = basic *extended
      .help = "The algorithm to compute mosaicity"
      .type = choice
    parameters {
      sigma_b = None
        .help = "Override the sigma_b value (degrees)"
        .type = float(value_min=0)
      sigma_m = None
        .help = "Override the sigma_m value (degrees)"
        .type = float(value_min=0)
    }
    filter {
      min_zeta = 0.05
        .help = "Filter reflections by min zeta"
        .type = float
    }
    fitting {
      scan_step = 5
        .help = "Space between profiles in degrees"
        .type = float
      grid_size = 5
        .help = "The size of the profile grid."
        .type = int
      threshold = 0.02
        .help = "The threshold to use in reference profile"
        .type = float
      grid_method = single *regular_grid circular_grid spherical_grid
        .help = "Select the profile grid method"
        .type = choice
      fit_method = *reciprocal_space detector_space
        .help = "The fitting method"
        .type = choice
      detector_space {
        deconvolution = False
          .help = "Do deconvolution in detector space"
          .type = bool
      }
    }
  }
  gaussian_rs
    .help = "An extension class implementing a reciprocal space gaussian"
            "profile model."
  {
    scan_varying = False
      .help = "Calculate a scan varying model"
      .type = bool
    min_spots
      .help = "if (total_reflections > overall or reflections_per_degree >"
              "per_degree) then do the profile modelling."
    {
      overall = 50
        .help = "The minimum number of spots needed to do the profile"
                "modelling"
        .type = int(value_min=0)
      per_degree = 20
        .help = "The minimum number of spots needed to do the profile"
                "modelling"
        .type = int(value_min=0)
    }
    sigma_m_algorithm = basic *extended
      .help = "The algorithm to compute mosaicity"
      .type = choice
    parameters {
      sigma_b = None
        .help = "Override the sigma_b value (degrees)"
        .type = float(value_min=0)
      sigma_m = None
        .help = "Override the sigma_m value (degrees)"
        .type = float(value_min=0)
    }
    filter {
      min_zeta = 0.05
        .help = "Filter reflections by min zeta"
        .type = float
    }
    fitting {
      scan_step = 5
        .help = "Space between profiles in degrees"
        .type = float
      grid_size = 5
        .help = "The size of the profile grid."
        .type = int
      threshold = 0.02
        .help = "The threshold to use in reference profile"
        .type = float
      grid_method = single *regular_grid circular_grid spherical_grid
        .help = "Select the profile grid method"
        .type = choice
      fit_method = *reciprocal_space detector_space
        .help = "The fitting method"
        .type = choice
      detector_space {
        deconvolution = False
          .help = "Do deconvolution in detector space"
          .type = bool
      }
    }
  }
}
prediction {
  d_min = None
    .help = "The maximum resolution limit"
    .type = float
  d_max = None
    .help = "The minimum resolution limit"
    .type = float
  margin = 1
    .help = "The margin to use to scan varying prediction"
    .type = int
  force_static = False
    .help = "For scan-varying prediction force scan-static prediction"
    .type = bool
  padding = 1.0
    .help = "The padding in degrees"
    .type = float(value_min=0)
}
significance_filter {
  enable = False
    .help = "If enabled, the significance filter will, for every experiment,"
            "find the highest resolution where the I/sigI remains above a"
            "certain point (controlled by isigi_cutoff)."
    .type = bool
  n_bins = 20
    .help = "Number of bins to use when examining resolution falloff"
    .type = int
  isigi_cutoff = 1.0
    .help = "I/sigI cutoff. Reflections in and past the first bin that falls"
            "below this cutoff will not be retained"
    .type = float
}
absorption_correction
  .multiple = True
{
  apply = False
    .help = "must be supplied as a user-defined function with a specific"
            "interface (not documented)"
    .type = bool
  algorithm = fuller_kapton other
    .help = "a specific absorption correction, or implementation thereof"
    .type = choice
  fuller_kapton {
    xtal_height_above_kapton_mm {
      value = 0.02
        .help = "height of the beam (or the irradiated crystal) above the"
                "kapton tape"
        .type = float
      sigma = 0.01
        .type = float
    }
    rotation_angle_deg {
      value = 1.15
        .help = "angle of the tape from vertical"
        .type = float
      sigma = 0.1
        .type = float
    }
    kapton_half_width_mm {
      value = 1.5875
        .help = "forward distance from irradiated crystal to edge of tape"
                "nearest detector"
        .type = float
      sigma = 0.5
        .type = float
    }
    kapton_thickness_mm {
      value = 0.05
        .help = "tape thickness"
        .type = float
      sigma = 0.005
        .type = float
    }
    smart_sigmas = False
      .help = "apply spot-specific sigma corrections using kapton param sigmas"
      .type = bool
    within_spot_sigmas = True
      .help = "turn this off to get a major speed-up"
      .type = bool
  }
}