This is how you load the term object from a hook preprocess term reference field and get the value of a custom field. In my case i want to get the custom color each category term has.
function mytheme_preprocess_field__node__field_custom(&$vars) {
//We can have more that one categories
$current_first_tid = $vars['element']['#object']->get('field_main_category')->first()->target_id;
$term_obj = Drupal::entityTypeManager()->getStorage('taxonomy_term')->load($current_first_tid);
$cat_color = $term_obj->get('field_category_color')->value;
//Make sure we always return a default color hex value
$vars['cat_color'] = isset($cat_color) ? $cat_color : '#cc0000';
}