<commit_before>############################################################################
# DIALS tutorial script - associated with data doi:10.5281/zenodo.10271
############################################################################

# first import the data - this reads the image headers and creates a file which
# describes the sweeps found therein. here there should only be one sweep found
# data should be passed on the command-line e.g. ./tutorial.sh /path/to/data
# or linked to the directory where the script is run as "data"

if [[ -z $1 ]]; then
  data_directory=./data
else
  data_directory=$1
fi

if [[ ! -f ${data_directory}/th_8_2_0001.cbf ]]; then
  echo "Data not found in directory: ${data_directory} -"
  echo "please download from doi:10.5281/zenodo.10271 and pass on command-line"
  echo "or create softlink from data directory to ./data"
  exit
fi

dials.import ${data_directory}/th_8_2_0*.cbf

# find spots on the images - this will take a while! also accept small spots
# in this case as the data are good (N.B. to explore DIALS options you can
# run dials.parameters - this shows you the entire parameter set which is
# available for DIALS)

dials.find_spots min_spot_size=3 datablock.json

# index these found spots - in this case the crystal is thaumatin which is known
# to be tetragonal, so impose appropriate lattice constraints in the indexing
# and refinement. can also specify unit cell here and also apply different
# indexing algorithms

dials.index space_group=P4 datablock.json strong.pickle \
  refinement.reflections.use_all_reflections=true

# run the refinement of the data - the indexing includes refinement so the result
# will be refined already with a static model - however here we would like to use
# a scan varying model so re-run the refinement (you should find that the R.M.S.
# deviations are a little lower following the scan varying refinement)

dials.refine experiments.json indexed.pickle \
  refinement.parameterisation.crystal.scan_varying=true \
  refinement.reflections.use_all_reflections=true

# now run the integration - complex choices of algorithms are shown here in
# terms of the peak measurement (fitrs) and background determination
# methods. pass reference reflections from indexing in to determine the
# profile parameters...

dials.integrate outlier.algorithm=null intensity.algorithm=fitrs \
  refined_experiments.json reference=indexed.pickle

# finally export the integrated measurements in an MTZ file - this should be
# properly formatted for immediate use in pointless / aimless

dials.export_mtz integrated.pickle refined_experiments.json integrated.mtz

# and as if to make a point, here is all we need to do is to sort the data with
# pointless and then scale the data (ignoring anomalous differences) to 1.3A,
# and run ctruncate for the intensity analysis...

pointless hklin integrated.mtz hklout sorted.mtz

aimless hklin sorted.mtz hklout scaled.mtz << eof
resolution 1.3
anomalous off
eof

ctruncate -hklin scaled.mtz -hklout truncated.mtz \
-colin '/*/*/[IMEAN,SIGIMEAN]'<commit_msg>Run integrate2; use parallel<commit_after>############################################################################
# DIALS tutorial script - associated with data doi:10.5281/zenodo.10271
############################################################################

# first import the data - this reads the image headers and creates a file which
# describes the sweeps found therein. here there should only be one sweep found
# data should be passed on the command-line e.g. ./tutorial.sh /path/to/data
# or linked to the directory where the script is run as "data"

if [[ -z $1 ]]; then
  data_directory=./data
else
  data_directory=$1
fi

if [[ ! -f ${data_directory}/th_8_2_0001.cbf ]]; then
  echo "Data not found in directory: ${data_directory} -"
  echo "please download from doi:10.5281/zenodo.10271 and pass on command-line"
  echo "or create softlink from data directory to ./data"
  exit
fi

dials.import ${data_directory}/th_8_2_0*.cbf

# find spots on the images - this will take a while! also accept small spots
# in this case as the data are good (N.B. to explore DIALS options you can
# run dials.parameters - this shows you the entire parameter set which is
# available for DIALS)

dials.find_spots min_spot_size=3 datablock.json \
  max_procs=$((`libtbx.show_number_of_processors` / 2))

# index these found spots - in this case the crystal is thaumatin which is known
# to be tetragonal, so impose appropriate lattice constraints in the indexing
# and refinement. can also specify unit cell here and also apply different
# indexing algorithms

dials.index space_group=P4 datablock.json strong.pickle \
  refinement.reflections.use_all_reflections=true

# run the refinement of the data - the indexing includes refinement so the result
# will be refined already with a static model - however here we would like to use
# a scan varying model so re-run the refinement (you should find that the R.M.S.
# deviations are a little lower following the scan varying refinement)

dials.refine experiments.json indexed.pickle \
  refinement.parameterisation.crystal.scan_varying=true \
  refinement.reflections.use_all_reflections=true

# now run the integration - complex choices of algorithms are shown here in
# terms of the peak measurement (fitrs) and background determination
# methods. pass reference reflections from indexing in to determine the
# profile parameters...

dials.integrate2 outlier.algorithm=null intensity.algorithm=fitrs \
  refined_experiments.json reference=indexed.pickle \
  max_proc=$((`libtbx.show_number_of_processors` / 2))


# finally export the integrated measurements in an MTZ file - this should be
# properly formatted for immediate use in pointless / aimless

dials.export_mtz integrated.pickle refined_experiments.json integrated.mtz

# and as if to make a point, here is all we need to do is to sort the data with
# pointless and then scale the data (ignoring anomalous differences) to 1.3A,
# and run ctruncate for the intensity analysis...

pointless hklin integrated.mtz hklout sorted.mtz

aimless hklin sorted.mtz hklout scaled.mtz << eof
resolution 1.3
anomalous off
eof

ctruncate -hklin scaled.mtz -hklout truncated.mtz \
-colin '/*/*/[IMEAN,SIGIMEAN]'
{'fraction_non_alphanumeric': 0.11279032771237671, 'fraction_numerical': 0.013044861597200128, 'mean_word_length': 4.588444444444445, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3370365', 'n_tokens_mistral': 1853, 'n_tokens_neox': 1733, 'n_words': 742}