# dials.integrate¶

## Introduction¶

This program is used to integrate the reflections on the diffraction images. It is called with an experiment list outputted from dials.index or dials.refine and a corresponding set of strong spots from which a profile model is calculated. The program will output a set of integrated reflections and an experiment list with additional profile model data. The data can be reintegrated using the same profile model by inputting this integrated_experiments.json file back into dials.integate.

Examples:

```
dials.integrate experiments.json indexed.pickle
dials.integrate experiments.json indexed.pickle output.reflections=integrated.pickle
dials.integrate experiments.json indexed.pickle profile.fitting=False
dials.integrate experiments.json indexed.pickle background.algorithm=glm
```

## Basic parameters¶

```
output {
experiments = 'integrated_experiments.json'
reflections = 'integrated.pickle'
phil = 'dials.integrate.phil'
log = 'dials.integrate.log'
debug_log = 'dials.integrate.debug.log'
report = None
include_bad_reference = False
}
scan_range = None
create_profile_model = True
exclude_images = None
verbosity = 1
integration {
lookup {
mask = None
}
block {
size = auto
units = *degrees radians frames
threshold = 0.99
force = False
max_memory_usage = 0.75
}
use_dynamic_mask = True
debug {
reference {
filename = "reference_profiles.pickle"
output = False
}
during = modelling *integration
output = False
separate_files = True
delete_shoeboxes = False
select = None
split_experiments = True
}
profile {
fitting = True
validation {
number_of_partitions = 1
min_partition_size = 100
}
}
overlaps_filter {
foreground_foreground {
enable = False
}
foreground_background {
enable = False
}
}
mp {
method = *none drmaa sge lsf pbs
njobs = 1
nproc = 1
}
summation {
detector_gain = 1
}
background {
algorithm = simple null median gmodel *glm
simple {
outlier {
algorithm = *null nsigma truncated normal plane tukey
}
model {
algorithm = constant2d *constant3d linear2d linear3d
}
min_pixels = 10
}
median {
}
gmodel {
robust {
algorithm = False
tuning_constant = 1.345
}
min_pixels = 10
model = None
}
glm {
robust {
tuning_constant = 1.345
}
model {
algorithm = constant2d *constant3d loglinear2d loglinear3d
}
min_pixels = 10
}
}
centroid {
algorithm = *simple
}
}
profile {
algorithm = gaussian_rs *gaussian_rs
gaussian_rs {
scan_varying = False
min_spots {
overall = 50
per_degree = 20
}
sigma_m_algorithm = basic *extended
parameters {
sigma_b = None
sigma_m = None
}
filter {
min_zeta = 0.05
}
fitting {
scan_step = 5
grid_size = 5
threshold = 0.02
grid_method = single *regular_grid circular_grid spherical_grid
fit_method = *reciprocal_space detector_space
detector_space {
deconvolution = False
}
}
}
gaussian_rs {
scan_varying = False
min_spots {
overall = 50
per_degree = 20
}
sigma_m_algorithm = basic *extended
parameters {
sigma_b = None
sigma_m = None
}
filter {
min_zeta = 0.05
}
fitting {
scan_step = 5
grid_size = 5
threshold = 0.02
grid_method = single *regular_grid circular_grid spherical_grid
fit_method = *reciprocal_space detector_space
detector_space {
deconvolution = False
}
}
}
}
prediction {
d_min = None
d_max = None
margin = 1
force_static = False
padding = 1.0
}
significance_filter {
enable = False
n_bins = 20
isigi_cutoff = 1.0
}
absorption_correction {
apply = False
algorithm = fuller_kapton other
fuller_kapton {
xtal_height_above_kapton_mm {
value = 0.02
sigma = 0.01
}
rotation_angle_deg {
value = 1.15
sigma = 0.1
}
kapton_half_width_mm {
value = 1.5875
sigma = 0.5
}
kapton_thickness_mm {
value = 0.05
sigma = 0.005
}
smart_sigmas = False
within_spot_sigmas = True
}
}
```

## Full parameter definitions¶

```
output {
experiments = 'integrated_experiments.json'
.help = "The experiments output filename"
.type = str
reflections = 'integrated.pickle'
.help = "The integrated output filename"
.type = str
phil = 'dials.integrate.phil'
.help = "The output phil file"
.type = str
log = 'dials.integrate.log'
.help = "The log filename"
.type = str
debug_log = 'dials.integrate.debug.log'
.help = "The debug log filename"
.type = str
report = None
.help = "The integration report filename (*.xml or *.json)"
.type = str
include_bad_reference = False
.help = "Include bad reference data including unindexed spots, and"
"reflections whose predictions are messed up in the reflection"
"table output. Reflections will have the 'bad_reference' flag set."
.type = bool
}
scan_range = None
.help = "Explicitly specify the images to be processed. Only applicable when"
"experiment list contains a single imageset."
.type = ints(size=2)
.multiple = True
create_profile_model = True
.help = "Create the profile model"
.type = bool
sampling
.expert_level = 1
{
reflections_per_degree = 50
.help = "The number of predicted reflections per degree of the sweep to"
"integrate."
.type = float(value_min=0)
minimum_sample_size = 1000
.help = "cutoff that determines whether subsetting of the input "
"prediction list is done"
.type = int
maximum_sample_size = None
.help = "The maximum number of predictions to integrate. Overrides"
"reflections_per_degree if that produces a larger sample size."
.type = int(value_min=1)
integrate_all_reflections = True
.help = "Override reflections_per_degree and integrate all predicted"
"reflections."
.type = bool
}
exclude_images = None
.help = "Exclude images from integration (e.g. 1,2,3,4,5 etc)"
.type = ints
verbosity = 1
.help = "The verbosity level"
.type = int(value_min=0)
integration {
lookup
.help = "Parameters specifying lookup file path"
{
mask = None
.help = "The path to the mask file."
.type = str
}
block {
size = auto
.help = "The block size in rotation angle (degrees)."
.type = float
units = *degrees radians frames
.help = "The units of the block size"
.type = choice
threshold = 0.99
.help = "For block size auto the block size is calculated by sorting"
"reflections by the number of frames they cover and then"
"selecting the block size to be 2*nframes[threshold] such that"
"100*threshold % of reflections are guarenteed to be fully"
"contained in 1 block"
.type = float(value_min=0, value_max=1)
force = False
.help = "If the number of processors is 1 and force is False, then the"
"number of blocks may be set to 1. If force is True then the"
"block size is always calculated."
.type = bool
max_memory_usage = 0.75
.help = "The maximum percentage of total physical memory to use for"
"allocating shoebox arrays."
.type = float(value_min=0, value_max=1)
}
use_dynamic_mask = True
.help = "Use dynamic mask if available"
.type = bool
debug {
reference {
filename = "reference_profiles.pickle"
.help = "The filename for the reference profiles"
.type = str
output = False
.help = "Save the reference profiles"
.type = bool
}
during = modelling *integration
.help = "Do debugging during modelling or integration"
.type = choice
output = False
.help = "Save shoeboxes after each processing task."
.type = bool
separate_files = True
.help = "If this is true, the shoeboxes are saved in separate files from"
"the output integrated.pickle file. This is necessary in most"
"cases since the amount of memory used by the shoeboxes is"
"typically greater than the available system memory. If,"
"however, you know that memory is not an issue, you can saved"
"the shoeboxes in the integrated.pickle file by setting this"
"option to False. This only works if the debug output is during"
"integrated and not modelling."
.type = bool
delete_shoeboxes = False
.help = "Delete shoeboxes immediately before saving files. This option"
"in combination with debug.output=True enables intermediate"
"processing steps to make use of shoeboxes."
.type = bool
select = None
.help = "A string specifying the selection. The string should be of the"
"form: select=${COLUMN}[<|<=|==|!=|>=|>]${VALUE}. In addition to"
"the items in the reflection table, the following implicit"
"columns are defined if the necessary data is there: "
"intensity.sum.i_over_sigma intensity.prf.i_over_sigma"
.type = reflection_table_selector
split_experiments = True
.help = "Split shoeboxes into different files"
.type = bool
}
profile {
fitting = True
.help = "Use profile fitting if available"
.type = bool
validation {
number_of_partitions = 1
.help = "The number of subsamples to take from the reference spots. If"
"the value is 1, then no validation is performed."
.type = int(value_min=1)
min_partition_size = 100
.help = "The minimum number of spots to use in each subsample."
.type = int(value_min=1)
}
}
filter
.expert_level = 1
{
min_zeta = 0.05
.help = "Filter the reflections by the value of zeta. A value of less"
"than or equal to zero indicates that this will not be used. A"
"positive value is used as the minimum permissable value."
.type = float(value_min=0, value_max=1)
max_shoebox_overlap = 1.0
.help = "Filter reflections whose shoeboxes are overlapped by greater"
"than the requested amount. Note that this is not the percentage"
"of the peak that is overlapped but rather the percentage of the"
"shoebox (background and foreground). This can be useful when"
"the detector is too close and many overlapping reflections are"
"predicted at high resolution causing memory issues."
.type = float(value_min=0, value_max=1)
ice_rings = False
.help = "Set the ice ring flags"
.type = bool
}
overlaps_filter {
foreground_foreground {
enable = False
.help = "Remove all spots in which neighbors' foreground impinges on"
"the spot's foreground"
.type = bool
}
foreground_background {
enable = False
.help = "Remove all spots in which neighbors' foreground impinges on"
"the spot's background"
.type = bool
}
}
mp {
method = *none drmaa sge lsf pbs
.help = "The cluster method to use"
.type = choice
njobs = 1
.help = "The number of cluster jobs to use"
.type = int(value_min=1)
nproc = 1
.help = "The number of processes to use per cluster job"
.type = int(value_min=1)
}
summation {
detector_gain = 1
.help = "Multiplier for variances after integration of still images. See"
"Leslie 1999."
.type = float
}
background
.help = " I n t e r f a c e f o r b a c k g r o u n d a l g o r i t"
"h m s . "
{
algorithm = simple null median gmodel *glm
.help = "The choice of algorithm"
.type = choice
simple
.help = "An extension class implementing simple background subtraction."
{
outlier
.help = "Outlier rejection prior to background fit"
{
algorithm = *null nsigma truncated normal plane tukey
.help = "The outlier rejection algorithm."
.type = choice
nsigma
.help = "Parameters for nsigma outlier rejector"
.expert_level = 1
{
lower = 3
.help = "Lower n sigma"
.type = float
upper = 3
.help = "Upper n sigma"
.type = float
}
truncated
.help = "Parameters for truncated outlier rejector"
.expert_level = 1
{
lower = 0.01
.help = "Lower bound"
.type = float
upper = 0.01
.help = "Upper bound"
.type = float
}
normal
.help = "Parameters for normal outlier rejector"
.expert_level = 1
{
min_pixels = 10
.help = "The minimum number of pixels to use in calculating the"
"background intensity."
.type = int
}
plane
.help = "Parameters for mosflm-like outlier rejector. This algorithm"
"is mainly used in conjunction with a linear 2d background."
.expert_level = 1
{
fraction = 1.0
.help = "The fraction of pixels to use in determining the initial"
"plane used for outlier rejection."
.type = float
n_sigma = 4.0
.help = "The number of standard deviations above the threshold"
"plane to use in rejecting outliers from background"
"calculation."
.type = float
}
tukey
.help = "Parameters for tukey outlier rejector"
.expert_level = 1
{
lower = 1.5
.help = "Lower IQR multiplier"
.type = float
upper = 1.5
.help = "Upper IQR multiplier"
.type = float
}
}
model
.help = "Background model"
{
algorithm = constant2d *constant3d linear2d linear3d
.help = "The choice of background model"
.type = choice
}
min_pixels = 10
.help = "The minimum number of pixels to compute the background"
.type = int(value_min=1)
}
median
.help = "An extension class implementing a median background algorithm."
{
}
gmodel
.help = "An extension class implementing a robust GLM background"
"algorithm."
{
robust {
algorithm = False
.help = "Use the robust algorithm"
.type = bool
tuning_constant = 1.345
.help = "The tuning constant for robust estimation"
.type = float
}
min_pixels = 10
.help = "The minimum number of pixels required"
.type = int(value_min=1)
model = None
.help = "The model filename"
.type = str
}
glm
.help = "An extension class implementing a robust GLM background"
"algorithm."
{
robust {
tuning_constant = 1.345
.help = "The tuning constant for robust estimation"
.type = float
}
model {
algorithm = constant2d *constant3d loglinear2d loglinear3d
.help = "The background model to fit"
.type = choice
}
min_pixels = 10
.help = "The minimum number of pixels required"
.type = int(value_min=1)
}
}
centroid
.help = " I n t e r f a c e f o r c e n t r o i d a l g o r i t h m"
"s . "
{
algorithm = *simple
.help = "The choice of algorithm"
.type = choice
}
}
profile
.help = "
T h e i n t e r f a c e d e f i n i t i o n f o r a "
"p r o f i l e m o d e l .
"
{
algorithm = gaussian_rs *gaussian_rs
.help = "The choice of algorithm"
.type = choice
gaussian_rs
.help = "An extension class implementing a reciprocal space gaussian"
"profile model."
{
scan_varying = False
.help = "Calculate a scan varying model"
.type = bool
min_spots
.help = "if (total_reflections > overall or reflections_per_degree >"
"per_degree) then do the profile modelling."
{
overall = 50
.help = "The minimum number of spots needed to do the profile"
"modelling"
.type = int(value_min=0)
per_degree = 20
.help = "The minimum number of spots needed to do the profile"
"modelling"
.type = int(value_min=0)
}
sigma_m_algorithm = basic *extended
.help = "The algorithm to compute mosaicity"
.type = choice
parameters {
sigma_b = None
.help = "Override the sigma_b value (degrees)"
.type = float(value_min=0)
sigma_m = None
.help = "Override the sigma_m value (degrees)"
.type = float(value_min=0)
}
filter {
min_zeta = 0.05
.help = "Filter reflections by min zeta"
.type = float
}
fitting {
scan_step = 5
.help = "Space between profiles in degrees"
.type = float
grid_size = 5
.help = "The size of the profile grid."
.type = int
threshold = 0.02
.help = "The threshold to use in reference profile"
.type = float
grid_method = single *regular_grid circular_grid spherical_grid
.help = "Select the profile grid method"
.type = choice
fit_method = *reciprocal_space detector_space
.help = "The fitting method"
.type = choice
detector_space {
deconvolution = False
.help = "Do deconvolution in detector space"
.type = bool
}
}
}
gaussian_rs
.help = "An extension class implementing a reciprocal space gaussian"
"profile model."
{
scan_varying = False
.help = "Calculate a scan varying model"
.type = bool
min_spots
.help = "if (total_reflections > overall or reflections_per_degree >"
"per_degree) then do the profile modelling."
{
overall = 50
.help = "The minimum number of spots needed to do the profile"
"modelling"
.type = int(value_min=0)
per_degree = 20
.help = "The minimum number of spots needed to do the profile"
"modelling"
.type = int(value_min=0)
}
sigma_m_algorithm = basic *extended
.help = "The algorithm to compute mosaicity"
.type = choice
parameters {
sigma_b = None
.help = "Override the sigma_b value (degrees)"
.type = float(value_min=0)
sigma_m = None
.help = "Override the sigma_m value (degrees)"
.type = float(value_min=0)
}
filter {
min_zeta = 0.05
.help = "Filter reflections by min zeta"
.type = float
}
fitting {
scan_step = 5
.help = "Space between profiles in degrees"
.type = float
grid_size = 5
.help = "The size of the profile grid."
.type = int
threshold = 0.02
.help = "The threshold to use in reference profile"
.type = float
grid_method = single *regular_grid circular_grid spherical_grid
.help = "Select the profile grid method"
.type = choice
fit_method = *reciprocal_space detector_space
.help = "The fitting method"
.type = choice
detector_space {
deconvolution = False
.help = "Do deconvolution in detector space"
.type = bool
}
}
}
}
prediction {
d_min = None
.help = "The maximum resolution limit"
.type = float
d_max = None
.help = "The minimum resolution limit"
.type = float
margin = 1
.help = "The margin to use to scan varying prediction"
.type = int
force_static = False
.help = "For scan-varying prediction force scan-static prediction"
.type = bool
padding = 1.0
.help = "The padding in degrees"
.type = float(value_min=0)
}
significance_filter {
enable = False
.help = "If enabled, the significance filter will, for every experiment,"
"find the highest resolution where the I/sigI remains above a"
"certain point (controlled by isigi_cutoff)."
.type = bool
n_bins = 20
.help = "Number of bins to use when examining resolution falloff"
.type = int
isigi_cutoff = 1.0
.help = "I/sigI cutoff. Reflections in and past the first bin that falls"
"below this cutoff will not be retained"
.type = float
}
absorption_correction
.multiple = True
{
apply = False
.help = "must be supplied as a user-defined function with a specific"
"interface (not documented)"
.type = bool
algorithm = fuller_kapton other
.help = "a specific absorption correction, or implementation thereof"
.type = choice
fuller_kapton {
xtal_height_above_kapton_mm {
value = 0.02
.help = "height of the beam (or the irradiated crystal) above the"
"kapton tape"
.type = float
sigma = 0.01
.type = float
}
rotation_angle_deg {
value = 1.15
.help = "angle of the tape from vertical"
.type = float
sigma = 0.1
.type = float
}
kapton_half_width_mm {
value = 1.5875
.help = "forward distance from irradiated crystal to edge of tape"
"nearest detector"
.type = float
sigma = 0.5
.type = float
}
kapton_thickness_mm {
value = 0.05
.help = "tape thickness"
.type = float
sigma = 0.005
.type = float
}
smart_sigmas = False
.help = "apply spot-specific sigma corrections using kapton param sigmas"
.type = bool
within_spot_sigmas = True
.help = "turn this off to get a major speed-up"
.type = bool
}
}
```